fix HCPCS parsing (rss xml no longer available)

OHDSI · Nov 20, 2019 · 8a1002c · 8a1002c
1 parent b6065c5
commit 8a1002c
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/working/packages/vocabulary_download/get_hcpcs.sql b/working/packages/vocabulary_download/get_hcpcs.sql
@@ -58,7 +58,8 @@ BEGIN
     --get credentials
     select vocabulary_auth, vocabulary_url, vocabulary_login, vocabulary_pass
     into pVocabulary_auth, pVocabulary_url, pVocabulary_login, pVocabulary_pass from devv5.vocabulary_access where vocabulary_id=pVocabularyID and vocabulary_order=1;
-
+
+    /*old version
     --parse RSS to get raw link to download page
     select t.link into pDownloadURL from (
       with url as (select http_content from py_http_get(url=>pVocabulary_url))
@@ -69,10 +70,14 @@ BEGIN
     ) as t
     where t.link like '%Alpha-Numeric-HCPCS-File%' and t.link not like '%orrections%'
     order by t.title desc limit 1;
+
     if not coalesce(pDownloadURL,'-') ~* '^https?://www.cms.gov/Medicare/Coding/(.+)Alpha.Numeric.HCPCS.File.*html$' then pErrorDetails:=coalesce(pDownloadURL,'-'); raise exception 'pDownloadURL (raw) is not valid'; end if;
     
     --getting fully working download link from page
     select substring(pDownloadURL,'^(https?://([^/]+))')||substring(http_content,'.+<h2>Downloads</h2>.+<a type="application/zip" href="(.+?)">\d{4} Alpha-Numeric HCPCS File') into pDownloadURL from py_http_get(url=>pDownloadURL,allow_redirects=>true);
+    */
+    pDownloadURL := SUBSTRING(pVocabulary_url,'^(https?://([^/]+))')||SUBSTRING(http_content,'<a href="(/Medicare/Coding/HCPCSReleaseCodeSets/Alpha-Numeric-HCPCS-Items/[[:digit:]]{4}-Alpha-Numeric-HCPCS-File)">') from py_http_get(url=>pVocabulary_url,allow_redirects=>true);
+    pDownloadURL := SUBSTRING(pVocabulary_url,'^(https?://([^/]+))')||SUBSTRING(http_content,'<a href="(/Medicare/Coding/HCPCSReleaseCodeSets/Downloads/[[:digit:]]{4}-Alpha-Numeric-HCPCS-File.zip)">') from py_http_get(url=>pDownloadURL,allow_redirects=>true);
     if not coalesce(pDownloadURL,'-') ~* '^(https?://www.cms.gov/Medicare/Coding)(.+)\.zip$' then pErrorDetails:=coalesce(pDownloadURL,'-'); raise exception 'pDownloadURL (full) is not valid'; end if;
 
     perform write_log (

diff --git a/working/packages/vocabulary_pack/CheckVocabularyUpdate.sql b/working/packages/vocabulary_pack/CheckVocabularyUpdate.sql
@@ -162,13 +162,16 @@ BEGIN
                 cVocabVer := 'Snomed Release '||to_char(cVocabDate,'YYYYMMDD');
             WHEN cVocabularyName = 'HCPCS'
             THEN
+              cVocabDate := TO_DATE(SUBSTRING(cVocabHTML,'<a href="/Medicare/Coding/HCPCSReleaseCodeSets/Alpha-Numeric-HCPCS-Items/([[:digit:]]{4})-Alpha-Numeric-HCPCS-File">')::int - 1 || '0101', 'yyyymmdd');
+              /*old version
               select TO_DATE ( (MAX (t.title) - 1) || '0101', 'yyyymmdd') into cVocabDate  From (
                 select 
                     unnest(xpath ('/rss/channel/item/title/text()', cVocabHTML::xml))::varchar::int title,
                     unnest(xpath ('/rss/channel/item/link/text()', cVocabHTML::xml)) ::varchar description 
               ) as t
               WHERE t.description LIKE '%Alpha-Numeric-HCPCS-File%' AND t.description NOT LIKE '%orrections%';
-              cVocabVer := to_char(cVocabDate,'YYYYMMDD')||' Alpha Numeric HCPCS File';
+              */
+              cVocabVer := to_char(cVocabDate + interval '1 year','YYYY')||' Alpha Numeric HCPCS File';
             WHEN cVocabularyName IN ('ICD9CM', 'ICD9PROC')
             THEN
                 cSearchString := '<a type="application/zip" href="/Medicare/Coding/ICD9ProviderDiagnosticCodes/Downloads';