diff --git a/ATC/load_stage.sql b/ATC/load_stage.sql index e74e42787..391ffe27f 100644 --- a/ATC/load_stage.sql +++ b/ATC/load_stage.sql @@ -14,26 +14,25 @@ * limitations under the License. * * Authors: Anna Ostropolets, Timur Vakhitov -* Date: 2018 +* Date: Jan 2020 **************************************************************************/ /* Prerequisites: -1. Get the latest file from source, name it class_drugs_scraper. The file should contain class code, class name, additional information. +1. Get the latest file FROM source, name it class_drugs_scraper. The file should contain class code, class name, additional information. For example, for ATC the file will be created in the following manner: SELECT id, atc_code AS class_code, atc_name AS class_name, ddd, u, adm_r, note, description, ddd_description FROM atc_drugs_scraper; 2. Prepare input tables (drug_concept_stage, internal_relationship_stage, relationship_to_concept) according to the general rules for drug vocabularies 3. Prepare the following tables: -- reference (represents the original code and concantinated code and its drug forms in RxNorm format) +- reference (represents the original code and concatenated code AND its drug forms INRxNorm format) - class_to_drug_manual (stores manual mappings, i.g. Insulins) -- ambiguous_class_ingredient (class, code, class_name, ingredients of ATC as ing, flag [ing for the main ingredient,with for additional ingredients,excl for those that should be excluded]). +- ambiguous_class_ingredient (class, code, class_name, ingredients of ATC AS ing, flag [ing for the main ingredient,with for additional ingredients,excl for those that should be excluded]). For groups of ingredients (e.g. antibiotics) list all the possible variations */ - --- 1. Update latest_update field to new date +-- 1. Update latest_UPDATE field to new date DO $_$ BEGIN PERFORM VOCABULARY_PACK.SetLatestUpdate( @@ -52,1003 +51,844 @@ TRUNCATE TABLE pack_content_stage; TRUNCATE TABLE drug_strength_stage; -- 3. Preliminary work --- 3.1 CREATE TABLE with combo drugs to be used later +-- 3.0 create table with manual work +DROP TABLE IF EXISTS class_to_drug_manual_tbd; +CREATE TABLE class_to_drug_manual_tbd AS + SELECT * FROM class_drugs_scraper +WHERE class_code ~ '^J07|^A10A' +AND length(class_code)=7; + +-- 3.1 create table with combo drugs to be used later DROP TABLE IF EXISTS class_1_comb; CREATE TABLE class_1_comb AS SELECT * FROM class_drugs_scraper -WHERE class_name ~ 'comb| and |diphtheria-|meningococcus|excl|derivate|other|with' +WHERE class_name ~ 'comb| AND |excl|derivate|other|with' AND length(class_code) = 7 - AND NOT class_name ~ 'decarboxylase inhibitor'; + -- AND NOT class_name ~ 'decarboxylase inhibitor' XXX + AND class_code NOT IN (SELECT class_code FROM class_to_drug_manual_tbd) -- manual XXX +; --- 3.2 create a table with aggregated -/RxE ingredients -DROP TABLE IF EXISTS rx_combo; -CREATE TABLE rx_combo AS -SELECT drug_concept_id, - string_agg(ingredient_concept_id::varchar, '-' order by ingredient_concept_id) AS i_combo -FROM drug_strength - JOIN concept ON concept_id = drug_concept_id and - concept_class_id IN ('Clinical Drug Form') -- 'Clinical Drug Comp' doesn exist -GROUP BY drug_concept_id +DELETE +FROM class_1_comb +WHERE class_code = 'S01XA20';--artificial tears AND other indifferent preparations, actually not combo + +-- create unified table with combinations, combos a+b +DROP TABLE IF EXISTS ambiguous_class_ingredient_tst; +CREATE TABLE ambiguous_class_ingredient_tst AS + -- a+b +SELECT class_code,class_name, concept_code_2, CASE WHEN rnk = 1 THEN 'ing' ELSE 'with' END AS flag, rnk +FROM ( + SELECT class_code, + concept_code_2, + class_name, + coalesce(rnk, CASE + WHEN rnk IS NULL + THEN rank() OVER (PARTITION BY concept_code_1, rnk ORDER BY concept_code_2) + 1 END) AS rnk + FROM ( + SELECT DISTINCT class_code, + i.concept_code_2, + class_name, + CASE + WHEN concept_code_2 = + regexp_replace(regexp_replace(class_name, ' and.*', ''), '\, .*', '') THEN 1 + ELSE NULL END AS rnk, + concept_code_1 + FROM class_1_comb + LEFT JOIN reference USING (class_code) + JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 + JOIN drug_concept_stage d + ON lower(d.concept_code) = lower(concept_code_2) AND concept_class_id = 'Ingredient' + WHERE class_name ~ ' AND ' + AND NOT class_name ~ 'excl|combinations of|derivate|other|with') a + ) a +UNION +-- complex combinations +SELECT class_code, class_name, ing, flag, rnk +FROM ambiguous_class_ingredient ; --- 3.3 Precise combos (a and b) -DROP TABLE IF EXISTS simple_comb; -CREATE TABLE simple_comb AS - with ing AS - (SELECT i.concept_code_1,i.concept_code_2, class_name,precedence,rtc.concept_id_2 AS ing +-- one strange parsing +UPDATE ambiguous_class_ingredient_tst +SET flag = 'ing', rnk = 1 +WHERE class_code='A11GB01' AND concept_code_2='ascorbic acid (vit c)'; + +UPDATE ambiguous_class_ingredient_tst +SET rnk = 2 +WHERE class_code='A11GB01' AND concept_code_2='calcium'; + + +DELETE +FROM ambiguous_class_ingredient_tst +WHERE (class_code, concept_code_2, flag) IN (SELECT a.class_code,a.concept_code_2,a.flag + FROM ambiguous_class_ingredient_tst a + JOIN ambiguous_class_ingredient_tst b + ON a.class_code = b.class_code AND a.concept_code_2 = b.concept_code_2 + WHERE a.flag = 'with' AND b.flag = 'ing'); + +-- insert pure combinations +DROP TABLE IF EXISTS pure_combos; +CREATE TABLE pure_combos + as + SELECT * FROM ( + SELECT class_code,i.concept_code_2, class_name, 'ing' AS flag, 1 AS rnk FROM class_1_comb - left JOIN reference using (class_code) + LEFT JOIN reference USING (class_code) JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d ON d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - WHERE class_name ~ ' and ' - AND NOT class_name ~ 'excl|combinations of|derivate|other|with' - ), - form AS - (SELECT i.concept_code_1,rtc.concept_id_2 AS form - FROM class_1_comb - left JOIN reference using (class_code) - JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d ON d.concept_code = concept_code_2 AND concept_class_id = 'Dose Form' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - WHERE class_name ~ ' and ' - AND NOT class_name ~ 'excl|combinations of|derivate|other|with' - ) - SELECT DISTINCT i.concept_code_1,i.concept_code_2, class_name, ing,form,precedence - FROM ing i - left JOIN form f ON i.concept_code_1 = f.concept_code_1; - --- 3.4 mapping 1 to 1 for those drugs that have ingredients with 2 or more mappings in relationship_to_concept -DROP TABLE IF EXISTS class_combo; -CREATE TABLE class_combo AS -SELECT concept_code_1, - string_agg(ing::varchar, '-' order by ing) AS i_combo -FROM simple_comb -WHERE concept_code_1 in - (SELECT concept_code_1 - FROM simple_comb - WHERE concept_code_2 in - (SELECT concept_code_2 - FROM (SELECT DISTINCT concept_code_2, ing FROM simple_comb) s - GROUP BY concept_code_2 - HAVING COUNT(1) = 1) - GROUP BY concept_code_1 - HAVING COUNT(1) = 2) -GROUP BY concept_code_1 + JOIN drug_concept_stage d ON lower(d.concept_code) = lower(concept_code_2) AND concept_class_id = 'Ingredient' + WHERE class_name IN('combinations','various combinations') +UNION + SELECT class_code, i.concept_code_2, class_name, 'with', 2 + FROM class_1_comb a + JOIN concept c ON regexp_replace(c.concept_code, '..$', '') = regexp_replace(a.class_code, '..$', '') and + c.concept_class_id = 'ATC 5th' -- getting siblings for combinations + JOIN internal_relationship_stage i ON c.concept_code = substring(i.concept_code_1,'\w+') + JOIN drug_concept_stage d ON lower(d.concept_code) = lower(i.concept_code_2) AND d.concept_class_id = 'Ingredient' + WHERE a.class_name IN('combinations','various combinations') +) a ; -INSERT INTO class_combo AS -SELECT concept_code_1, - string_agg(ing::varchar, '-' order by ing) AS i_combo -FROM simple_comb -WHERE concept_code_1 in - (SELECT concept_code_1 - FROM simple_comb - WHERE concept_code_2 in - (SELECT concept_code_2 - FROM (SELECT DISTINCT concept_code_2, ing FROM simple_comb) s - GROUP BY concept_code_2 - HAVING COUNT(1) = 1) - GROUP BY concept_code_1 - HAVING COUNT(1) = 3) -AND class_name LIKE '%, % and %' -GROUP BY concept_code_1 -; --- 3.4.1 create the actual table, start with ATCs with forms -DROP TABLE IF EXISTS class_to_drug_1; -CREATE TABLE class_to_drug_1 AS -SELECT concept_code_1, class_name, c2.concept_id, c2.concept_name, c2.concept_class_id -FROM class_combo - JOIN simple_comb using (concept_code_1) - JOIN rx_combo using (i_combo) - JOIN concept c ON c.concept_id = drug_concept_id - JOIN concept_relationship cr - ON concept_id_1 = c.concept_id AND cr.invalid_reason is null AND relationship_id = 'RxNorm has dose form' - AND concept_id_2 = form - JOIN concept c2 ON concept_id_1 = c2.concept_id; - --- 3.4.2 insert the ATCs without forms -INSERT INTO class_to_drug_1 -SELECT concept_code_1, class_name, c.concept_id, c.concept_name, c.concept_class_id -FROM class_combo - JOIN simple_comb using (concept_code_1) - JOIN rx_combo using (i_combo) - JOIN concept c ON c.concept_id = drug_concept_id -WHERE concept_code_1 NOT IN (SELECT concept_code_1 FROM class_to_drug_1) - AND concept_code_1 NOT like '% %' +UPDATE pure_combos v +SET rnk = 1, flag = 'ing' +FROM ( +SELECT class_code, min(concept_code_2) OVER (PARTITION BY class_code ORDER BY concept_code_2) AS min_code +FROM pure_combos +WHERE class_code NOT IN (SELECT class_code FROM pure_combos WHERE rnk=1) + ) a +WHERE (v.class_code =a.class_code AND v.concept_code_2 = a.min_code) ; --- 3.4.3 introducing precedence -INSERT INTO class_to_drug_1 -with class_comb AS - (with hold AS (SELECT * - FROM simple_comb s - WHERE NOT exists(SELECT 1 - FROM simple_comb s2 - WHERE s2.concept_code_2 = s.concept_code_2 AND s2.precedence > 1) -- we hold - ) - SELECT h.concept_code_1, - s.class_name, - case when h.ing > s.ing then s.ing || '-' || h.ing else h.ing || '-' || s.ing end AS i_combo, - h.form - FROM hold h - JOIN simple_comb s ON h.concept_code_1 = s.concept_code_1 AND h.ing != s.ing - WHERE h.concept_code_1 NOT IN (SELECT concept_code_1 FROM class_to_drug_1)) -SELECT concept_code_1, class_name, c.concept_id, c.concept_name, c.concept_class_id -FROM class_comb - JOIN rx_combo using (i_combo) - JOIN concept c ON c.concept_id = drug_concept_id - JOIN concept_relationship cr - ON concept_id_1 = c.concept_id AND cr.invalid_reason is null AND relationship_id = 'RxNorm has dose form' - AND concept_id_2 = form - JOIN concept c2 ON concept_id_2 = c2.concept_id -; -INSERT INTO class_to_drug_1 -with class_comb AS - (with hold AS (SELECT * - FROM simple_comb s - WHERE NOT exists(SELECT 1 - FROM simple_comb s2 - WHERE s2.concept_code_2 = s.concept_code_2 AND s2.precedence > 1) -- we hold - ) - SELECT h.concept_code_1, - s.class_name, - case when h.ing > s.ing then s.ing || '-' || h.ing else h.ing || '-' || s.ing end AS i_combo, - h.form - FROM hold h - JOIN simple_comb s ON h.concept_code_1 = s.concept_code_1 AND h.ing != s.ing - WHERE h.concept_code_1 NOT IN (SELECT concept_code_1 FROM class_to_drug_1)) -SELECT concept_code_1, class_name, c.concept_id, c.concept_name, c.concept_class_id -FROM class_comb - JOIN rx_combo using (i_combo) - JOIN concept c ON c.concept_id = drug_concept_id -WHERE concept_code_1 NOT like '% %' --w/o forms -; - - -- temporary solution for wierd ATCs that got mapped incorrectly -DELETE -FROM class_to_drug_1 -WHERE class_name like '%,%and%' - AND class_name NOT like '%,%,%and%' - AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT like '% / % / %'; - --- 3.5 usual combos (a, combinations) -DROP TABLE IF EXISTS verysimple_comb; -CREATE TABLE verysimple_comb AS - with ing AS - (SELECT i.concept_code_1,i.concept_code_2, class_name,rtc.concept_id_2 AS ing, 'ing' AS flag +DROP TABLE IF EXISTS verysimple_comb; --XXX +CREATE TABLE verysimple_comb + as + SELECT * FROM ( + SELECT class_code,i.concept_code_2, class_name, 'ing' AS flag, 1 AS rnk FROM class_1_comb - left JOIN reference using (class_code) + LEFT JOIN reference USING (class_code) JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 JOIN drug_concept_stage d ON d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - WHERE class_name ~ 'comb' - AND NOT class_name ~ 'excl| AND |combinations of|derivate|other|with' - ), - form AS - (SELECT i.concept_code_1,rtc.concept_id_2 AS form - FROM class_1_comb - left JOIN reference using (class_code) - JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d ON d.concept_code = concept_code_2 AND concept_class_id = 'Dose Form' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - WHERE class_name ~ 'comb' - AND NOT class_name ~ 'excl| AND |combinations of|derivate|other|with' - ), - addit AS - (SELECT i.concept_code_1, i.concept_code_2, class_name, rtc.concept_id_2 AS ing, 'with' AS flag + WHERE class_name ~ '(combinations of|in combination)' + AND NOT class_name ~ ' AND |derivate|other drugs|^other |with' AND class_name NOT IN ('combinations','various combinations' + ) +UNION + SELECT class_code, i.concept_code_2, class_name, 'with', 2 FROM class_1_comb a - left JOIN reference r using (class_code) - JOIN concept c ON regexp_replace(c.concept_code, '..$', '') = regexp_replace(a.class_code, '..$', '') and - c.concept_class_id = 'ATC 5th' - JOIN internal_relationship_stage i ON coalesce(r.concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d ON d.concept_code = concept_code_2 AND d.concept_class_id = 'Ingredient' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - ) - SELECT DISTINCT i.concept_code_1,i.concept_code_2, class_name, ing,form, flag - FROM (SELECT * - FROM ing - union all - SELECT * - FROM addit) i - left JOIN form f ON i.concept_code_1 = f.concept_code_1 - order by i.concept_code_1; - -DROP TABLE IF EXISTS class_to_drug_2; -CREATE TABLE class_to_drug_2 AS - with secondary_table AS ( + JOIN concept c ON regexp_replace(c.concept_code, '...$', '') = regexp_replace(a.class_code, '...$', '') and + c.concept_class_id = 'ATC 5th' -- getting siblings for combinations + JOIN internal_relationship_stage i ON c.concept_code = substring(i.concept_code_1,'\w+') + JOIN drug_concept_stage d ON lower(d.concept_code) = lower(i.concept_code_2) AND d.concept_class_id = 'Ingredient' + WHERE class_name ~ '(combinations of|in combination)' + AND NOT class_name ~ ' AND |derivate|other drugs|^other |with' AND class_name NOT IN ('combinations','various combinations') +) a +; + +-- combinations got all 2 ranks AS no primary ingredient exists (combination of, which can also go into pure_combos table) +UPDATE verysimple_comb v +set rnk = 1, flag = 'ing' +FROM ( +SELECT class_code, min(concept_code_2) OVER (PARTITION BY class_code ORDER BY concept_code_2) AS min_code +FROM verysimple_comb +WHERE class_code NOT IN (SELECT class_code FROM verysimple_comb WHERE rnk=1) + ) a +WHERE (v.class_code =a.class_code AND v.concept_code_2 = a.min_code) +; + +INSERT INTO ambiguous_class_ingredient_tst +SELECT DISTINCT class_code, class_name, concept_code_2, flag, rnk +FROM verysimple_comb +UNION +SELECT DISTINCT class_code, class_name, concept_code_2, flag, rnk +FROM pure_combos +; + +CREATE INDEX ambiguous_class_ingredient_test ON ambiguous_class_ingredient_tst (class_code, concept_code_2, flag); + +-- 3.2 create a table with aggregated RxE ingredients +DROP TABLE IF EXISTS rx_combo; +CREATE TABLE rx_combo AS +SELECT drug_concept_id, + string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo +FROM devv5.drug_strength + JOIN concept ON concept_id = drug_concept_id AND + concept_class_id IN ('Clinical Drug Form', 'Ingredient') -- 'Clinical Drug Comp' doesn't exist +GROUP BY drug_concept_id +; + +DROP TABLE IF EXISTS full_combo; +CREATE TABLE full_combo +AS +WITH hold AS ( +SELECT a.*, concept_id_2, precedence +FROM ambiguous_class_ingredient_tst a +JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE flag = 'ing'), + ing AS ( +SELECT a.*, concept_id_2, precedence +FROM ambiguous_class_ingredient_tst a +JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE flag = 'with' AND rnk = 2 + ), + ing2 AS ( +SELECT a.*, concept_id_2, precedence +FROM ambiguous_class_ingredient_tst a +JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE flag = 'with' AND rnk = 3 + ), + ing3 AS ( +SELECT a.*, concept_id_2, precedence +FROM ambiguous_class_ingredient_tst a +JOIN relationship_to_concept rtc ON rtc.concept_code_1 = concept_code_2 +WHERE flag = 'with' AND rnk = 4 + ) +SELECT hold.class_code, hold.class_name, hold.concept_id_2||COALESCE('-' || ing.concept_id_2, '')||COALESCE('-' || ing2.concept_id_2, '')||COALESCE('-' || ing3.concept_id_2, '') AS i_combo + FROM hold +JOIN ing USING (class_code) +LEFT JOIN ing2 USING (class_code) +LEFT JOIN ing3 USING (class_code) +ORDER BY class_code +; + +INSERT INTO full_combo +SELECT DISTINCT * +FROM permutations; + +CREATE INDEX i_full_combo ON full_combo (class_code, i_combo); + +DROP TABLE IF EXISTS full_combo_reodered; +CREATE TABLE full_combo_reodered + as +with a AS (SELECT class_code,class_name, i_combo, rank() OVER (PARTITION BY class_code ORDER BY i_combo) AS rnk + FROM (SELECT DISTINCT * FROM full_combo) a), + b AS (SELECT class_code,class_name, rnk, cast(unnest(string_to_array(i_combo, '-')) AS int) AS ing + FROM a) +SELECT class_code, class_name, string_agg(ing::varchar, '-' ORDER BY ing) AS i_combo +FROM b +GROUP BY class_code, class_name, rnk; -- to array + +CREATE INDEX i_full_combo_reodered ON full_combo_reodered (class_code, i_combo); + + +DROP TABLE IF EXISTS full_combo_with_form; +CREATE TABLE full_combo_with_form +AS +SELECT class_code, class_name, i_combo, concept_id_2 +FROM full_combo_reodered +JOIN internal_relationship_stage irs on class_code = substring (irs.concept_code_1,'\w+') +JOIN drug_concept_stage dc on dc.concept_code = irs.concept_code_2 AND dc.concept_class_id = 'Dose Form' +JOIN relationship_to_concept rtc on rtc.concept_code_1 = irs.concept_code_2 + +UNION ALL + +SELECT f.class_code, class_name, i_combo, null -- take from reference those that do not have forms +FROM full_combo_reodered f +JOIN reference r on r.class_code = f.class_code +WHERE r.concept_code = r.class_code +; + +CREATE INDEX i_full_combo_with_form ON full_combo_with_form (class_code, i_combo,concept_id_2); + +DROP TABLE IF EXISTS class_to_drug; --747 +CREATE TABLE class_to_drug +as + with rxnorm AS ( SELECT a.concept_id, a.concept_name,a.concept_class_id,a.vocabulary_id, c.concept_id_2,r.i_combo FROM rx_combo r JOIN concept a ON r.drug_concept_id = a.concept_id JOIN concept_relationship c ON c.concept_id_1 = a.concept_id WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id like 'RxNorm%'--temporary remove RXE - AND a.invalid_reason is null + AND a.vocabulary_id LIKE 'RxNorm%' + AND a.invalid_reason IS NULL AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason is null - ), - primary_table AS ( - SELECT v.concept_code_1, - v.form, - v.class_name, - case - when v.ing > v2.ing then cast(v2.ing AS varchar) || '-' || cast(v.ing AS varchar) - else cast(v.ing AS varchar) || '-' || cast(v2.ing AS varchar) end AS class_combo - FROM verysimple_comb v - JOIN verysimple_comb v2 ON v.concept_code_1 = v2.concept_code_1 AND v.flag = 'ing' AND v2.flag = 'with' - WHERE v.ing != v2.ing) - SELECT DISTINCT p.concept_code_1, class_name, s.concept_id, s.concept_name, s.concept_class_id - FROM primary_table p - JOIN secondary_table s - ON s.concept_id_2 = p.form - AND s.i_combo = p.class_combo + AND c.invalid_reason IS NULL + ) + SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id + FROM full_combo_with_form f + JOIN rxnorm r on r.i_combo = f.i_combo AND r.concept_id_2 = f.concept_id_2 ; -INSERT INTO class_to_drug_2 -with primary_table AS ( - SELECT v.concept_code_1, - v.form, - v.class_name, - case - when v.ing > v2.ing then cast(v2.ing AS varchar) || '-' || cast(v.ing AS varchar) - else cast(v.ing AS varchar) || '-' || cast(v2.ing AS varchar) end AS class_combo - FROM verysimple_comb v - JOIN verysimple_comb v2 ON v.concept_code_1 = v2.concept_code_1 AND v.flag = 'ing' AND v2.flag = 'with' - WHERE v.ing != v2.ing) -SELECT DISTINCT p.concept_code_1, class_name, c.concept_id, c.concept_name, c.concept_class_id -FROM primary_table p - JOIN rx_combo r ON p.class_combo = r.i_combo - JOIN concept c ON c.concept_id = r.drug_concept_id AND c.concept_class_id = 'Clinical Drug Form' and - c.vocabulary_id like 'RxNorm%' -WHERE p.concept_code_1 NOT like '% %'--exclude classes with known forms +-- adding everything we can for combinations +DROP TABLE IF EXISTS combo_not_limited_to_higherATC; +CREATE TABLE combo_not_limited_to_higherATC +AS + WITH combo AS ( --250 + SELECT c.class_code, c.class_name, concept_id_2 + FROM class_1_comb c + JOIN internal_relationship_stage i ON c.class_code = substring(i.concept_code_1, '\w+') + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 + WHERE c.class_name ~ '(, combinations)|(in combination with other drugs)' + AND NOT c.class_name ~ 'with|and thiazides|and other diuretics' -- gives errors INC07BB52 AND C07CB53 if removed + ) + SELECT DISTINCT class_code, class_name, drug_concept_id + FROM rx_combo + JOIN combo on i_combo ~ cast(concept_id_2 AS VARCHAR) AND i_combo LIKE '%-%' -- at least two ingredients + WHERE not exists(SELECT 1 + FROM ambiguous_class_ingredient_tst a2 + JOIN relationship_to_concept rtc2 ON rtc2.concept_code_1 = a2.concept_code_2 + WHERE a2.class_code = combo.class_code + AND a2.flag = 'excl' + AND i_combo ~ cast(rtc2.concept_id_2 AS VARCHAR)) ; --- 3.6 ATC combos with exclusions -DROP TABLE IF EXISTS compl_combo; -CREATE TABLE compl_combo AS - with hold AS ( - SELECT * - FROM ambiguous_class_ingredient d - JOIN relationship_to_concept ON concept_code_1 = ing AND flag = 'ing' --we hold - WHERE NOT exists(SELECT 1 - FROM ambiguous_class_ingredient d2 - WHERE d.class_code = d2.class_code - AND d.ing = d2.ing - AND precedence > 1) -- we hold, exclude multiple for now - ), - excl AS ( - SELECT * - FROM ambiguous_class_ingredient d - JOIN relationship_to_concept ON concept_code_1 = ing AND flag = 'excl' --we exclude - ), - additional AS ( - SELECT * - FROM ambiguous_class_ingredient d - JOIN relationship_to_concept ON concept_code_1 = ing AND flag = 'with' --we add - ) - SELECT DISTINCT r.concept_code AS concept_code_1, - h.class_name, - drug_concept_id AS concept_id, - concept_name, - concept_class_id - FROM hold h - JOIN reference r ON r.class_code = h.class_code - left JOIN additional a ON h.class_code = a.class_code - left JOIN excl e ON h.class_code = e.class_code - JOIN rx_combo ON i_combo ~ cast(h.concept_id_2 AS varchar) AND i_combo like '%-%' - JOIN concept c ON c.concept_id = drug_concept_id - WHERE (a.class_code is NOT null AND i_combo ~ cast(a.concept_id_2 AS varchar)) - or (e.class_code is NOT null AND NOT i_combo ~ cast(e.concept_id_2 AS varchar)) + WITH rxnorm AS ( + SELECT a.concept_id, a.concept_name,a.concept_class_id,a.vocabulary_id, c.concept_id_2,r.class_code, r.class_name + FROM combo_not_limited_to_higherATC r + JOIN concept a ON r.drug_concept_id = a.concept_id + JOIN concept_relationship c ON c.concept_id_1 = a.concept_id + WHERE a.concept_class_id = 'Clinical Drug Form' + AND a.vocabulary_id LIKE 'RxNorm%' + AND a.invalid_reason IS NULL + AND relationship_id = 'RxNorm has dose form' + AND c.invalid_reason IS NULL + ) +INSERT INTO class_to_drug +SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id +FROM rxnorm +JOIN internal_relationship_stage i ON substring(i.concept_code_1, '\w+') = class_code +JOIN relationship_to_concept rtc ON i.concept_code_2 = rtc.concept_code_1 +WHERE rxnorm.concept_id_2 = rtc.concept_id_2 ; --- 3.6.1 with no excluded -INSERT INTO compl_combo -with hold AS ( - SELECT * - FROM ambiguous_class_ingredient d - JOIN relationship_to_concept ON concept_code_1 = ing AND flag = 'ing' --we hold - WHERE NOT exists(SELECT 1 - FROM ambiguous_class_ingredient d2 - WHERE d.class_code = d2.class_code - AND d.ing = d2.ing - AND precedence > 1) -- we hold, exclude multiple for now -), - excl AS ( - SELECT * - FROM ambiguous_class_ingredient d - JOIN relationship_to_concept ON concept_code_1 = ing AND flag = 'excl' --we exclude - ), - additional AS ( - SELECT * - FROM ambiguous_class_ingredient d - JOIN relationship_to_concept ON concept_code_1 = ing AND flag = 'with' --we add - ) - -SELECT DISTINCT r.concept_code AS concept_code_1, - h.class_name, - drug_concept_id AS concept_id, - concept_name, - concept_class_id -FROM hold h - JOIN reference r ON r.class_code = h.class_code - JOIN additional a ON h.class_code = a.class_code - left JOIN excl e ON e.class_code = h.class_code - JOIN rx_combo ON i_combo ~ cast(h.concept_id_2 AS varchar) AND i_combo ~ cast(a.concept_id_2 AS varchar) and - i_combo like '%-%' - JOIN concept c ON c.concept_id = drug_concept_id -WHERE e.class_name is null +INSERT INTO class_to_drug +SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id +FROM combo_not_limited_to_higherATC +JOIN concept ON concept_id = drug_concept_id +JOIN reference r USING (class_code) +WHERE r.concept_code NOT LIKE '% %' ; --- 3.6.2 with -DROP TABLE IF EXISTS class_to_drug_3; -CREATE TABLE class_to_drug_3 AS -SELECT c.* -FROM compl_combo c - JOIN internal_relationship_stage i ON i.concept_code_1 = c.concept_code_1 - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 - JOIN concept_relationship cr ON cr.concept_id_1 = c.concept_id - AND relationship_id = 'RxNorm has dose form' AND cr.invalid_reason is null -WHERE cr.concept_id_2 = rtc.concept_id_2 - AND class_name like '%with%' + +--insert all forms for those ATC that do not specify forms --856 +WITH rxnorm AS ( + SELECT a.concept_id, a.concept_name,a.concept_class_id,a.vocabulary_id, c.concept_id_2,r.i_combo + FROM rx_combo r + JOIN concept a ON r.drug_concept_id = a.concept_id + JOIN concept_relationship c ON c.concept_id_1 = a.concept_id + WHERE a.concept_class_id = 'Clinical Drug Form' + AND a.vocabulary_id LIKE 'RxNorm%' + AND a.invalid_reason IS NULL + AND relationship_id = 'RxNorm has dose form' + AND c.invalid_reason IS NULL +) +INSERT +INTO class_to_drug +SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id +FROM full_combo_with_form f + JOIN rxnorm r ON r.i_combo = f.i_combo +WHERE f.concept_id_2 IS NULL ; --- 3.6.3 inserting everything that goes without a form -INSERT INTO class_to_drug_3 -SELECT * -FROM compl_combo -WHERE concept_code_1 NOT like '% %' - AND class_name like '%with%'; + +--Combinations of propranolol AND hydralazine or dihydralazine are classified INC07FX01. [combinations with other drugs] XXX -- 3.6.4 start removing incorrectly assigned combo based ON WHO rank +-- XXX https://www.whocc.no/atc_ddd_index/?code=N02BA51&showdescription=yes -- zero rank (no official rank is present) -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'M03BA73|M03BA72|N02AC74|M03BB72|N02BB52|M03BB73|M09AA72|N02AB72|N02BB72|N02BA77' +SELECT * --XXX +FROM class_to_drug +WHERE class_code ~ 'M03BA73|M03BA72|N02AC74|M03BB72|N02BB52|M03BB73|M09AA72|N02AB72|N02BB72|N02BA77' AND concept_name ~* - 'Salicylamide|Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Methadone|etamizole|Ergotamine' + 'Salicylamide|Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Methadone|etamizole|Ergotamine'--acetylsalicylic ; --starts the official rank -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'N02BB74' - AND concept_name ~* 'Salicylamide|Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' -; -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'N02BB51' - AND concept_name ~* 'Salicylamide|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' +DELETE --309 +FROM class_to_drug +WHERE class_code ~ 'N02BB74|N02BB54' + AND concept_name ~* 'Phenazone|Salicylamide|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' ; -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'N02BA75' +DELETE --294 +FROM class_to_drug +WHERE class_code ~ 'N02BA75|N02BA55' AND concept_name ~* 'Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' ; -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'N02BB71' +DELETE --228 +FROM class_to_drug +WHERE class_code ~ 'N02BB71|N02BB51' AND concept_name ~* 'Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' ; -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'N02BA71' +DELETE --55 +FROM class_to_drug +WHERE class_code ~ 'N02BA71|N02BA51' AND concept_name ~* 'Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin' ; -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'N02BE71' +DELETE -- 16 +FROM class_to_drug +WHERE class_code ~ 'N02BE71|N02BE51' AND concept_name ~* 'Dipyrocetyl|Bucetin|Phenacetin' ; -DELETE -FROM class_to_drug_3 -WHERE concept_code_1 ~ 'N02' +DELETE --522 +FROM class_to_drug +WHERE class_code ~ '^N02' AND concept_name ~ 'Codeine' AND NOT class_name ~ 'codeine'; +--3.7.3 atenolol AND other diuretics, combinations, one of a kind +SELECT * +FROM class_to_drug +WHERE class_code ~ 'C07CB53' + AND concept_name NOT LIKE '%/%/%'; --XXX why isn't it catching 3 ingredients? sometimes only 2 + + +-- PPI AND aspirin DELETE -FROM class_to_drug_3 -WHERE concept_id IN --removing duplicates - (SELECT concept_id FROM class_to_drug_1); +FROM class_to_drug +WHERE class_code ~ 'N02BA51' + AND concept_name ~* 'Omeprazole|Pantoprazole|Rabeprazol'; --- 3.7 class codes of type a+b excl. c -DROP TABLE IF EXISTS class_to_drug_4; -CREATE TABLE class_to_drug_4 AS -SELECT c.* -FROM compl_combo c - JOIN internal_relationship_stage i ON i.concept_code_1 = c.concept_code_1 - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 - JOIN concept_relationship cr ON cr.concept_id_1 = c.concept_id - AND relationship_id = 'RxNorm has dose form' AND cr.invalid_reason is null -WHERE cr.concept_id_2 = rtc.concept_id_2 - AND class_name NOT like '%with%' -; +-- add ranks +ALTER TABLE class_to_drug +ADD "order" INT; --- 3.7.1 inserting everything that goes without a form -INSERT INTO class_to_drug_4 -SELECT * -FROM compl_combo -WHERE concept_code_1 NOT like '% %' - AND class_name NOT like '%with%'; +UPDATE class_to_drug +SET "order"=2 +WHERE class_name ~ ' AND ' +and not class_name ~ 'excl|combinations of|derivate|with'; --- 3.7.2 start removing incorrectly assigned combo based ON WHO rank --- zero rank -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'M03BA53|M03BA52|N02AC54|M03BB52|M03BB53|N02AB52|N02BB52|N02CA52|N02BA57' - AND concept_name ~* 'Salicylamide|Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Methadone' -; ---starts the official rank -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02BB54|N02BB52|M03BB52' - AND concept_name ~* 'Salicylamide|Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Ergotamine|Dipyrone' -; -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02BA55' - AND concept_name ~* 'Phenazone|Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Ergotamine|Metamizole' -; -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02BB51' - AND concept_name ~* 'Aspirin|Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Ergotamine|Metamizole' -; -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02BA51' - AND concept_name ~* 'Acetaminophen|Dipyrocetyl|Bucetin|Phenacetin|Ergotamine|Metamizole' -; -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02BE51|N02BB53' - AND concept_name ~* 'Dipyrocetyl|Bucetin|Phenacetin|Ergotamine|Metamizole' -; -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02AC54' - AND concept_name ~* 'Dipyrone'; +UPDATE class_to_drug +SET "order"=3 +WHERE -- class_name ~ 'excl|comb|derivate|other|with' AND -- removed since there exclusions INnames + "order" IS NULL; -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02' - AND concept_name ~ 'Codeine' - AND NOT class_name ~ 'codeine'; +-- contraceptive packs +INSERT INTO class_to_drug +SELECT class_code, class_name, c.concept_id,c.concept_name,"order" +FROM class_to_drug ctd +JOIN devv5.concept_ancestor ON ctd.concept_id = ancestor_concept_id +JOIN concept c ON descendant_concept_id = c.concept_id +WHERE class_code ~ 'G03FB|G03AB' + AND c.concept_class_id IN ('Clinical Pack'); -DELETE -FROM class_to_drug_4 -WHERE concept_id IN --removing duplicates - (SELECT concept_id FROM class_to_drug_1); +DELETE FROM class_to_drug +WHERE class_code ~ 'G03FB|G03AB' + AND concept_class_id NOT IN ('Clinical Pack'); ---3.7.3 atenolol AND other diuretics, combinations, one of a kind -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'C07CB53|C07DB01' - AND concept_name NOT like '%/%/%'; -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'C07CB03|C07CB53' - AND concept_name like '%/%/%'; --- PPI AND aspirin -DELETE -FROM class_to_drug_4 -WHERE concept_code_1 ~ 'N02BA51' - AND concept_name ~* 'Omeprazole|Pantoprazole|Rabeprazol'; - --- 3.8 Simlpe class codes that have dose forms -DROP TABLE IF EXISTS primary_table; -CREATE TABLE primary_table AS - with ing AS - (SELECT i.concept_code_1,class_name,rtc.concept_id_2 AS ing - FROM class_drugs_scraper - LEFT JOIN reference using (class_code) - JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d ON d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - WHERE class_name !~'combination|and|^other| with|mening|diphtheria' - ), - form AS - (SELECT i.concept_code_1,rtc.concept_id_2 AS form +-- insert mono-ATC codes +DROP TABLE IF EXISTS mono_ing; +CREATE TABLE mono_ing as +SELECT DISTINCT class_code, class_name, concept_id_2 AS ing_id FROM class_drugs_scraper - LEFT JOIN reference using (class_code) - JOIN internal_relationship_stage i ON coalesce(concept_code, class_code) = concept_code_1 - JOIN drug_concept_stage d ON d.concept_code = concept_code_2 AND concept_class_id = 'Dose Form' - JOIN relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - WHERE class_name !~'combination| and |^other| with|mening|diphtheria' - ) - SELECT DISTINCT i.concept_code_1, class_name, ing,form - FROM ing i - LEFT JOIN form f ON i.concept_code_1 = f.concept_code_1; - -DROP TABLE IF EXISTS class_to_drug_5; -CREATE TABLE class_to_drug_5 AS - with secondary_table AS ( - SELECT a.concept_id, - a.concept_name, - a.concept_class_id, - a.vocabulary_id, - c.concept_id_2 AS sform, - b.ingredient_concept_id AS sing - FROM concept a - JOIN drug_strength b ON b.drug_concept_id = a.concept_id - JOIN concept_relationship c ON c.concept_id_1 = a.concept_id +JOIN internal_relationship_stage i on class_code = substring(concept_code_1,'\w+') +JOIN drug_concept_stage d on lower(d.concept_code) = lower(i.concept_code_2) AND concept_class_id = 'Ingredient' +JOIN relationship_to_concept rtc on rtc.concept_code_1 = i.concept_code_2 + WHERE length(class_code)=7 +; + +DELETE FROM mono_ing +WHERE class_code IN(SELECT class_code FROM class_1_comb) +or class_code IN(SELECT class_code FROM class_to_drug_manual_tbd) +; + + +INSERT INTO class_to_drug --587434 +with form AS ( + SELECT class_code, class_name, ing_id, concept_id_2 AS form_id + FROM mono_ing + JOIN internal_relationship_stage i on class_code = substring(i.concept_code_1, '\w+') + JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Dose Form' + JOIN relationship_to_concept rtc on rtc.concept_code_1 = i.concept_code_2 +) +SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id,4 +FROM form +JOIN rx_combo r on i_combo = cast(ing_id AS varchar) --XXX ingr?? +JOIN concept a ON r.drug_concept_id = a.concept_id +JOIN concept_relationship c ON c.concept_id_1 = a.concept_id AND form_id = c.concept_id_2 WHERE a.concept_class_id = 'Clinical Drug Form' - AND a.vocabulary_id like 'RxNorm%'--temporary remove RXE - AND a.invalid_reason is null + AND a.vocabulary_id LIKE 'RxNorm%' + AND a.invalid_reason IS NULL AND relationship_id = 'RxNorm has dose form' - AND c.invalid_reason is null - AND NOT exists(SELECT 1 - FROM drug_strength d - WHERE d.drug_concept_id = b.drug_concept_id - AND d.ingredient_concept_id != b.ingredient_concept_id) -- excluding combos - ) - SELECT DISTINCT p.concept_code_1, class_name, s.concept_id, s.concept_name, s.concept_class_id - FROM primary_table p, - secondary_table s - WHERE s.sform = p.form - AND s.sing = p.ing + AND c.invalid_reason IS NULL ; + +INSERT INTO class_to_drug --785 +SELECT DISTINCT class_code, class_name, c.concept_id, c.concept_name, c.concept_class_id, 5 +FROM mono_ing m +JOIN internal_relationship_stage i on class_code = substring(concept_code_1,'\w+') +JOIN concept c on m.ing_id = c.concept_id + WHERE i.concept_code_1 NOT LIKE '% %' +; + +INSERT INTO class_to_drug +SELECT class_code, class_name, concept_id, concept_name, concept_class_id,1 +FROM class_to_drug_manual; + + + +-- XXX 5 should surpass all -- 3.8.1 manually excluded drugs based ON Precise Ingredients -INSERT INTO class_to_drug_5 -SELECT 'B02BD11','catridecacog', concept_id, concept_name, concept_class_id + +DELETE +FROM class_to_drug +WHERE class_code IN ('B02BD14','B02BD11') +and concept_class_id = 'Ingredient'; + +INSERT INTO class_to_drug +SELECT 'B02BD11','catridecacog', concept_id, concept_name, concept_class_id,1 FROM concept -WHERE vocabulary_id like 'RxNorm%' AND concept_name like 'coagulation factor XIII a-subunit (recombinant)%' and +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE 'coagulation factor XIII a-subunit (recombinant)%' AND standard_concept = 'S' - or concept_id = 35603348 -- the whole hierarchy + OR concept_id = 35603348 -- the whole hierarchy ; -INSERT INTO class_to_drug_5 -SELECT 'B02BD14','susoctocog alfa', concept_id, concept_name, concept_class_id +INSERT INTO class_to_drug +SELECT 'B02BD14','susoctocog alfa', concept_id, concept_name, concept_class_id,1 FROM concept -WHERE vocabulary_id like 'RxNorm%' and - concept_name like 'antihemophilic factor, porcine B-domain truncated recombinant%' AND standard_concept = 'S' - or concept_id IN (35603348, 44109089) -- the whole hierarchy +WHERE vocabulary_id LIKE 'RxNorm%' and + concept_name LIKE 'antihemophilic factor, porcine B-domain truncated recombinant%' AND standard_concept = 'S' + OR concept_id IN (35603348, 44109089) -- the whole hierarchy ; + DELETE -FROM class_to_drug_5 -WHERE concept_code_1 = 'B02BD14' - AND concept_name like '%Tretten%' --catridecacog +FROM class_to_drug +WHERE class_code = 'A02BA07' + AND concept_class_id = 'Clinical Drug Form'; --Tritec +INSERT INTO class_to_drug +SELECT 'A02BA07','ranitidine bismuth citrate', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Tritec%' AND + standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' ; -DELETE -FROM class_to_drug_5 -WHERE class_name = 'amino acids'; --- 3.9 Class codes that do NOT have forms (ingredients only) -DROP TABLE IF EXISTS class_to_drug_6; -CREATE TABLE class_to_drug_6 AS - with secondary_table AS ( - SELECT a.concept_id, a.concept_name,a.concept_class_id,a.vocabulary_id, b.ingredient_concept_id AS sing - FROM concept a - JOIN drug_strength b - ON b.drug_concept_id = a.concept_id - WHERE a.concept_class_id = 'Ingredient' - AND a.invalid_reason is null - AND a.vocabulary_id = 'RxNorm%'--temporary remove RXE - AND NOT exists(SELECT 1 - FROM drug_strength d - WHERE d.drug_concept_id = b.drug_concept_id - AND d.ingredient_concept_id != b.ingredient_concept_id) -- excluding combos - ) - SELECT DISTINCT p.concept_code_1, class_name, s.concept_id, s.concept_name, s.concept_class_id - FROM primary_table p, - secondary_table s - WHERE s.sing = p.ing - AND p.form is null - AND p.concept_code_1 NOT IN (SELECT concept_code FROM reference WHERE concept_code != class_code)-- exclude drugs that should have forms (will remain unmapped) - ; - --- 4. Start final assembly, insert one-by-one going from the most precise class (a+b) to the simplest one (a, ingredient) --- TODO: G03FB|G03AB --- hardcoded combinations - --- 4.1 -DROP TABLE IF EXISTS class_to_rx_descendant; -CREATE TABLE class_to_rx_descendant -AS -SELECT DISTINCT substring(concept_code_1, '\w+') AS class_code, - class_name, - c.concept_id, - c.concept_name, - c.concept_code, - c.concept_class_id, - '1' AS order -FROM class_to_drug_1 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c - ON c.concept_id = descendant_concept_id AND vocabulary_id like 'RxNorm%' AND c.standard_concept = 'S'; --- 4.2.1 DELETE -FROM class_to_rx_descendant -WHERE class_code ~ 'G03FB|G03AB' - AND concept_class_id NOT like '%Pack%'; - --- 4.2.2 combinations from ATC 4th, need to be fixed afterwards -DELETE -FROM class_to_rx_descendant -WHERE class_name NOT LIKE '% / %' -AND class_code ~ 'S01CB|G03EK|G03CC|D10AA|D07XC|D07XB|D07XA' -; - --- 4.3 -INSERT INTO class_to_rx_descendant -SELECT DISTINCT substring(concept_code_1, '\w+'), - class_name, - c.concept_id, - c.concept_name, - c.concept_code, - c.concept_class_id, - '2' AS order -FROM class_to_drug_2 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c - ON c.concept_id = descendant_concept_id AND c.vocabulary_id like 'RxNorm%' AND c.standard_concept = 'S' -WHERE descendant_concept_id NOT IN (SELECT concept_id FROM class_to_rx_descendant); - --- 4.4 -INSERT INTO class_to_rx_descendant -SELECT DISTINCT substring(concept_code_1, '\w+'), - class_name, - c.concept_id, - c.concept_name, - c.concept_code, - c.concept_class_id, - '3' AS order -FROM class_to_drug_3 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c - ON c.concept_id = descendant_concept_id AND c.vocabulary_id like 'RxNorm%' AND c.standard_concept = 'S' -WHERE descendant_concept_id NOT IN (SELECT concept_id FROM class_to_rx_descendant); - --- 4.5 -INSERT INTO class_to_rx_descendant -SELECT DISTINCT substring(concept_code_1, '\w+'), - class_name, - c.concept_id, - c.concept_name, - c.concept_code, - c.concept_class_id, - '4' AS order -FROM class_to_drug_4 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c - ON c.concept_id = descendant_concept_id AND c.vocabulary_id like 'RxNorm%' AND c.standard_concept = 'S' -WHERE descendant_concept_id NOT IN (SELECT concept_id FROM class_to_rx_descendant); - ---4.6 -INSERT INTO class_to_rx_descendant -SELECT DISTINCT substring(concept_code_1, '\w+'), - class_name, - c.concept_id, - c.concept_name, - c.concept_code, - c.concept_class_id, - '5' AS order -FROM class_to_drug_5 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c - ON c.concept_id = descendant_concept_id AND c.vocabulary_id like 'RxNorm%' AND c.standard_concept = 'S' - JOIN drug_strength d ON d.drug_concept_id = c.concept_id -WHERE descendant_concept_id NOT IN (SELECT concept_id FROM class_to_rx_descendant) - AND NOT exists - (SELECT 1 - FROM concept c2 - JOIN devv5.concept_ancestor ca2 - ON ca2.ancestor_concept_id = c2.concept_id AND c2.concept_class_id = 'Ingredient' - AND c2.vocabulary_id like 'RxNorm%' - WHERE ca2.descendant_concept_id = d.drug_concept_id - AND c2.concept_id != d.ingredient_concept_id) -- excluding combos -; --- 4.7 working with packs -with a AS ( - SELECT DISTINCT a.*, - c.concept_id AS pack_id, - d.drug_concept_id, - d.ingredient_concept_id - FROM class_to_drug_5 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c ON c.concept_id = descendant_concept_id AND c.vocabulary_id like 'RxNorm%' - AND c.standard_concept = 'S' AND c.concept_class_id IN ('Clinical Pack ', 'Branded Pack') - JOIN concept_relationship cr - ON cr.concept_id_1 = c.concept_id AND cr.invalid_reason is null AND cr.relationship_id = 'Contains' - JOIN drug_strength d ON d.drug_concept_id = cr.concept_id_2 - WHERE descendant_concept_id NOT IN (SELECT concept_id - FROM сlass_to_rx_descendant) -), - b AS ( - SELECT DISTINCT concept_code_1, - class_name, - concept_id, - concept_name, - concept_class_id, - pack_id, - string_agg(ingredient_concept_id::varchar, '-' order by ingredient_concept_id) AS i_combo - FROM a - GROUP BY concept_code_1,class_name,concept_id,concept_name,concept_class_id, pack_id - ), - c AS ( - SELECT DISTINCT b.concept_code_1, - b.class_name, - b.concept_id, - b.concept_name, - b.concept_class_id, - b.i_combo, - pack_id, - string_agg(ca.ancestor_concept_id:: varchar, '-' order by ca.ancestor_concept_id) AS i_combo_2 - FROM b - JOIN devv5.concept_ancestor ca ON b.concept_id = ca.descendant_concept_id - JOIN concept c ON c.concept_id = ca.ancestor_concept_id AND c.concept_class_id = 'Ingredient' - GROUP BY b.concept_code_1, b.class_name, b.concept_id, b.concept_name, b.concept_class_id, b.i_combo, - ca.descendant_concept_id,pack_id - ) -INSERT -INTO class_to_rx_descendant -(class_code, class_name, concept_id, concept_name, concept_code, concept_class_id, "order") -SELECT DISTINCT substring(concept_code_1, '\w+'), - class_name, - pack_id, - cc.concept_name, - cc.concept_code, - cc.concept_class_id, - '5' -FROM c - JOIN concept cc ON cc.concept_id = pack_id -WHERE i_combo = i_combo_2 - OR i_combo = i_combo_2 || '-' || i_combo_2 +FROM class_to_drug -- choriogonadotropin alfa +WHERE class_code = 'G03GA08'; +INSERT INTO class_to_drug +SELECT 'G03GA08','choriogonadotropin alfa', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%choriogonadotropin alfa%' and + standard_concept = 'S' AND concept_class_id = 'Clinical Drug Form' ; --- 4.8 -INSERT INTO class_to_rx_descendant -SELECT DISTINCT substring(concept_code_1, '\w+'), - class_name, - c.concept_id, - c.concept_name, - c.concept_code, - c.concept_class_id, - '6' AS order -FROM class_to_drug_6 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c - ON c.concept_id = descendant_concept_id AND c.vocabulary_id LIKE 'RxNorm%' AND c.standard_concept = 'S' - JOIN drug_strength d ON d.drug_concept_id = c.concept_id -WHERE descendant_concept_id NOT IN (SELECT concept_id FROM class_to_rx_descendant) - AND NOT exists - (SELECT 1 - FROM concept c2 - JOIN devv5.concept_ancestor ca2 - ON ca2.ancestor_concept_id = c2.concept_id AND c2.concept_class_id = 'Ingredient' - AND c2.vocabulary_id LIKE 'RxNorm%' - WHERE ca2.descendant_concept_id = d.drug_concept_id - AND c2.concept_id != d.ingredient_concept_id) -- excluding combos +DELETE +FROM class_to_drug +WHERE class_code='N05AF02'; -- clopentixol +INSERT INTO class_to_drug +SELECT 'N05AF02','clopenthixol', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Sordinol|Ciatyl' and + standard_concept = 'S' AND concept_class_id = 'Branded Drug Form' ; --- 4.9 working with packs -WITH a AS ( - SELECT DISTINCT a.*, - c.concept_id AS pack_id, - d.drug_concept_id, - d.ingredient_concept_id - FROM class_to_drug_6 a - JOIN devv5.concept_ancestor ON ancestor_concept_id = a.concept_id - JOIN concept c ON c.concept_id = descendant_concept_id AND c.vocabulary_id LIKE 'RxNorm%' - AND c.standard_concept = 'S' AND c.concept_class_id IN ('Clinical Pack ', 'Branded Pack') - JOIN concept_relationship cr - ON cr.concept_id_1 = c.concept_id AND cr.invalid_reason is null AND cr.relationship_id = 'Contains' - JOIN drug_strength d ON d.drug_concept_id = cr.concept_id_2 - WHERE descendant_concept_id NOT IN (SELECT concept_id - FROM сlass_to_rx_descendant) -), - b AS ( - SELECT DISTINCT concept_code_1, - class_name, - concept_id, - concept_name, - concept_class_id, - pack_id, - string_agg(ingredient_concept_id::varchar, '-' ORDER BY ingredient_concept_id) AS i_combo - FROM a - GROUP BY concept_code_1,class_name,concept_id,concept_name,concept_class_id, pack_id - ), - c AS ( - SELECT DISTINCT b.concept_code_1, - b.class_name, - b.concept_id, - b.concept_name, - b.concept_class_id, - b.i_combo, - pack_id, - string_agg(ca.ancestor_concept_id:: varchar, '-' ORDER BY ca.ancestor_concept_id) AS i_combo_2 - FROM b - JOIN devv5.concept_ancestor ca ON b.concept_id = ca.descendant_concept_id - JOIN concept c ON c.concept_id = ca.ancestor_concept_id AND c.concept_class_id = 'Ingredient' - GROUP BY b.concept_code_1, b.class_name, b.concept_id, b.concept_name, b.concept_class_id, b.i_combo, - ca.descendant_concept_id,pack_id - ) -INSERT INTO class_to_rx_descendant -(class_code, class_name, concept_id, concept_name, concept_code, concept_class_id, "order") -SELECT DISTINCT substring(concept_code_1, '\w+'), - class_name, - pack_id, - cc.concept_name, - cc.concept_code, - cc.concept_class_id, - '6' -FROM c - JOIN concept cc ON cc.concept_id = pack_id -WHERE i_combo = i_combo_2 - or i_combo = i_combo_2 || '-' || i_combo_2 +DELETE +FROM class_to_drug +WHERE class_code IN('D07AB02','D07BB04'); -- hydrocortisone butyrate + combo that so far doesn't exist +INSERT INTO class_to_drug +SELECT 'D07AB02','hydrocortisone butyrate', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name ~* 'Hydrocortisone butyrate' and + standard_concept = 'S' ; --- 4.10 DELETE -FROM class_to_rx_descendant -WHERE class_name LIKE '%insulin%'; -INSERT INTO class_to_rx_descendant -SELECT DISTINCT class_code, - class_name, - c.concept_id, - c.concept_name, - c.concept_code, - c.concept_class_id, - '7' AS order -FROM class_to_drug_manual m - JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = m.concept_id - JOIN concept c ON c.concept_id = ca.descendant_concept_id - JOIN drug_strength d ON d.drug_concept_id = c.concept_id +FROM class_to_drug +WHERE class_code = 'C01DA05'; +INSERT INTO class_to_drug +SELECT 'C01DA05','pentaerithrityl tetranitrate', concept_id, concept_name, concept_class_id,1 +FROM concept +WHERE vocabulary_id LIKE 'RxNorm%' AND concept_name LIKE '%Pentaerythritol Tetranitrate%' and + standard_concept = 'S' AND concept_class_id = 'Clinical Drug Comp' -- forms do not exist ; --- 4.11 fix packs -INSERT INTO class_to_rx_descendant -SELECT DISTINCT substring(concept_code_1, '\w+'), class_name,c.concept_id, c.concept_name, c.concept_code, c.concept_class_id, 1 -FROM class_to_drug_1 f - JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = cast(f.concept_id AS int) - JOIN devv5.concept c ON c.concept_id = descendant_concept_id AND c.concept_class_id LIKE '%Pack%' -WHERE f.concept_code_1 ~ 'G03FB|G03AB'; -- packs - --- 4.12 DELETE -FROM class_to_rx_descendant -WHERE class_code ~ 'G03FB|G03AB' - AND concept_class_id IN ('Clinical Drug Form', 'Ingredient'); - +FROM class_to_drug +WHERE class_code = 'B02BD14' + AND concept_name LIKE '%Tretten%'; --catridecacog DELETE -FROM class_to_rx_descendant -WHERE class_name like '%and estrogen%' -- if there are regular estiol/estradiol/EE - AND concept_id IN (SELECT concept_id FROM сlass_to_rx_descendant GROUP BY concept_id HAVING COUNT(1) > 1); +FROM class_to_drug +WHERE class_code IN ('B02BD14','B02BD11') + AND "order" = 4 +; --- 4.13 temporary solution (same as before) DELETE -FROM class_to_rx_descendant -WHERE class_name like '%,%,%and%' - AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT like '%/%/%/%'; +FROM class_to_drug -- XXX +WHERE class_name = 'amino acids'; DELETE -FROM class_to_rx_descendant -WHERE class_name like '%,%and%' - AND class_name NOT like '%,%,%and%' - AND NOT class_name ~* 'comb|other|whole root|selective' - AND concept_name NOT like '% / % / %'; - +FROM class_to_drug +WHERE class_name LIKE '%,%and%' + AND class_name NOT LIKE '%,%,%and%' + AND NOT class_name ~* 'comb|other|whole root|SELECTive' + AND concept_name NOT LIKE '% / % / %'; + +--combinations FROM ATC 4th, need to be fixed afterwards DELETE -FROM class_to_rx_descendant +FROM class_to_drug WHERE class_name NOT LIKE '% / %' -AND class_code ~ 'S01CB|G03EK|G03CC|D10AA|D07XC|D07XB|D07XA' -; +AND class_code ~ 'S01CB|G03EK|G03CC|D10AA' -- D10AA smth for acne, unclear; S01CB with whatever you want, G% is strange hormones +; --- 4.14 repeate all the steps to get the table without ancestor (just the basic entities) -DROP TABLE IF EXISTS class_to_drug; -CREATE TABLE class_to_drug +DROP TABLE IF EXISTS pack_all; +CREATE TABLE pack_all AS -SELECT DISTINCT substring(concept_code_1, '\w+') AS class_code, - a.class_name, - a.concept_id, - a.concept_name, - a.concept_code_1, - a.concept_class_id, - '1' AS order -FROM class_to_drug_1 a -; -INSERT INTO class_to_drug -SELECT DISTINCT substring(concept_code_1, '\w+'), - a.class_name, - a.concept_id, - a.concept_name, - a.concept_code_1, - a.concept_class_id, - '2' AS order -FROM class_to_drug_2 a -WHERE concept_code_1 NOT IN - (SELECT class_code FROM class_to_drug) - AND a.concept_id NOT IN - (SELECT concept_id FROM class_to_drug) + WITH a AS ( + SELECT concept_id_1, + drug_concept_id, + string_agg(ingredient_concept_id::VARCHAR, '-' ORDER BY ingredient_concept_id) AS i_combo + FROM drug_strength + JOIN concept_relationship r + on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL + JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' + GROUP BY drug_concept_id,concept_id_1), + b as + (SELECT class_code, class_name, r.concept_code_1, r.concept_id_2 + FROM class_1_comb + JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code + JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' + JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 + WHERE class_name LIKE '% AND %'), + c AS (SELECT class_code, r.concept_code_1, r.concept_id_2 + FROM class_1_comb + JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code + JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' + JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 + WHERE class_name LIKE '% AND %') + SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name + FROM a + JOIN a aa on aa.concept_id_1 = a.concept_id_1 + JOIN concept cc on concept_id = a.concept_id_1 + JOIN b on cast(b.concept_id_2 AS varchar) = aa.i_combo + JOIN c on cast(c.concept_id_2 AS varchar) = a.i_combo + WHERE a.drug_concept_id != aa.drug_concept_id + AND b.class_code = c.class_code + AND b.concept_code_1 != c.concept_code_1 ; -INSERT INTO class_to_drug -SELECT DISTINCT substring(concept_code_1, '\w+'), - a.class_name, - a.concept_id, - a.concept_name, - a.concept_code_1, - a.concept_class_id, - '3' AS order -FROM class_to_drug_3 a -WHERE concept_code_1 NOT IN - (SELECT class_code FROM class_to_drug) + +-- XXX remove 3 components FROM pack_all +SELECT * +FROM pack_all; + +DROP TABLE IF EXISTS pack_temp; +CREATE TABLE pack_temp +as +with a AS (SELECT concept_id_1, + string_agg(ingredient_concept_id::varchar, '-' ORDER BY ingredient_concept_id) AS i_combo + FROM drug_strength + JOIN concept_relationship r on drug_concept_id = concept_id_2 AND relationship_id = 'Contains' AND r.invalid_reason IS NULL + JOIN concept c on c.concept_id = concept_id_1 AND concept_class_id = 'Clinical Pack' + GROUP BY concept_id_1), + b AS ( +SELECT DISTINCT class_code, class_name, r.concept_code_1, r.concept_id_2 +FROM class_1_comb +JOIN internal_relationship_stage i on substring(i.concept_code_1, '\w+') = class_code +JOIN drug_concept_stage d on d.concept_code = concept_code_2 AND concept_class_id = 'Ingredient' +JOIN relationship_to_concept r on r.concept_code_1 = i.concept_code_2 +WHERE class_name LIKE '%, combinations') +SELECT DISTINCT cc.concept_id, cc.concept_name, cc.concept_class_id, b.class_code, b.class_name +FROM a +JOIN b on i_combo ~ cast(b.concept_id_2 AS varchar) +and i_combo!=cast(b.concept_id_2 AS varchar) +JOIN concept cc on concept_id_1 = concept_id ; -INSERT INTO class_to_drug -SELECT DISTINCT substring(concept_code_1, '\w+'), - a.class_name, - a.concept_id, - a.concept_name, - a.concept_code_1, - a.concept_class_id, - '4' AS order -FROM class_to_drug_4 a -WHERE concept_code_1 NOT IN - (SELECT class_code FROM class_to_drug); -INSERT INTO class_to_drug -SELECT DISTINCT substring(concept_code_1, '\w+'), - a.class_name, - a.concept_id, - a.concept_name, - a.concept_code_1, - a.concept_class_id, - '5' AS order -FROM class_to_drug_5 a -WHERE concept_code_1 NOT IN - (SELECT class_code FROM class_to_drug); -INSERT INTO class_to_drug -SELECT DISTINCT substring(concept_code_1, '\w+'), - a.class_name, - a.concept_id, - a.concept_name, - a.concept_code_1, - a.concept_class_id, - '6' AS order -FROM class_to_drug_6 a -WHERE concept_code_1 NOT IN - (SELECT class_code FROM class_to_drug); +INSERT INTO pack_all +with a AS ( + SELECT p.concept_id, p.concept_name, p.concept_class_id, class_code,class_name, r.concept_id_2 + FROM pack_temp p + JOIN internal_relationship_stage on substring(concept_code_1, '\w+') = class_code + JOIN drug_concept_stage d on concept_code_2 = concept_code AND d.concept_class_id = 'Dose Form' + JOIN relationship_to_concept r on r.concept_code_1 = concept_code_2), + b AS (SELECT p.*, r2.concept_id_2 + FROM pack_temp p + JOIN devv5.concept_ancestor r on concept_id = descendant_concept_id + JOIN concept_relationship r2 + on r2.concept_id_1 = ancestor_concept_id AND r2.invalid_reason IS NULL and + relationship_id = 'RxNorm has dose form') +SELECT a.concept_id, a.concept_name, a.concept_class_id, a.class_code, a.class_name +FROM a +JOIN b USING (concept_id, concept_id_2) +UNION +SELECT p.* +FROM pack_temp p +JOIN reference r on concept_code = p.class_code +; -DELETE -FROM class_to_drug -WHERE class_name like '%insulin%'; INSERT INTO class_to_drug -SELECT DISTINCT class_code, - m.class_name, - m.concept_id, - m.concept_name, - c.concept_code, - m.concept_class_id, - '7' AS order -FROM class_to_drug_manual m - JOIN concept c using (concept_id) -; +(class_code, class_name, concept_id, concept_name, concept_class_id, "order") +SELECT DISTINCT class_code, class_name, concept_id, concept_name, concept_class_id, 4 +FROM pack_all; + +-- 4.11 fix packs INSERT INTO class_to_drug -SELECT substring(concept_code_1, '\w+'), f.class_name, c.concept_id, c.concept_name, c.concept_code, c.concept_class_id, f.order +SELECT DISTINCT class_code, class_name,c.concept_id, c.concept_name,c.concept_class_id, 4 FROM class_to_drug f - JOIN devv5.concept_ancestor ca - ON ca.ancestor_concept_id = f.concept_id + JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = cast(f.concept_id AS int) JOIN devv5.concept c ON c.concept_id = descendant_concept_id AND c.concept_class_id LIKE '%Pack%' -WHERE class_code ~ 'G03FB|G03AB'; -- packs +WHERE f.class_code ~ 'G03FB|G03AB'; -- packs +-- 4.12 DELETE FROM class_to_drug WHERE class_code ~ 'G03FB|G03AB' AND concept_class_id IN ('Clinical Drug Form', 'Ingredient'); -DELETE +SELECT * FROM class_to_drug -WHERE class_name NOT LIKE '% / %' -AND class_code ~ 'S01CB|G03EK|G03CC|D10AA|D07XC|D07XB|D07XA' -; +WHERE class_name LIKE '%and estrogen%' -- if there are regular estiol/estradiol/EE + AND concept_id IN (SELECT concept_id + FROM class_to_drug GROUP BY concept_id HAVING COUNT(1) > 1); + +-- working with duplicates to remove a AND b vs a/b, comb +DELETE FROM class_to_drug +WHERE (class_code,concept_id) IN( +SELECT b.class_code, b.concept_id FROM class_to_drug a +JOIN class_to_drug b on a.concept_id = b.concept_id +WHERE a."order" =1 AND b."order" =2); + + +-- 4.14 Solution for the first run: for inambiguous ATC classes (those that classify an ingredient through only one class) +-- we relate this ATC class to the entire group of drugs that have this ingredient. +drop table interim; +CREATE TABLE interim +as +with a AS (SELECT DISTINCT class_code, class_name, c.concept_id,c.concept_name +FROM class_to_drug crd +JOIN devv5.concept_ancestor on crd.concept_id = descendant_concept_id +JOIN concept c on c.concept_id = ancestor_concept_id AND c.concept_class_id = 'Ingredient') +SELECT * +FROM (SELECT count (*) OVER (PARTITION BY trim(regexp_replace(class_name, '\(.*\)','')) ) AS cnt, class_code, class_name, a.concept_id,a.concept_name -- regexp for (vit C) +FROM a ) a +WHERE cnt=1 +and class_code NOT IN (SELECT class_code FROM class_to_drug_manual) +; -DELETE -FROM class_to_drug -WHERE class_name like '%and estrogen%' -- if there are regular estiol/estradiol/EE - AND concept_id IN (SELECT concept_id FROM class_to_drug GROUP BY concept_id HAVING COUNT(1) > 1); +drop table interim_2; +CREATE TABLE interim_2 as +with a AS ( +SELECT DISTINCT class_code, class_name, c.concept_id,c.concept_name +FROM class_to_drug crd +JOIN devv5.concept_ancestor on crd.concept_id = descendant_concept_id +JOIN concept c on c.concept_id = ancestor_concept_id AND c.concept_class_id = 'Ingredient' + ), +b AS (SELECT concept_id FROM a +GROUP BY concept_id having count(1)=1), +c AS (SELECT a.class_code, a.class_name, a.concept_id FROM a JOIN b USING(concept_id) + ) +SELECT c.* +FROM c +JOIN interim USING(class_code) +WHERE class_code NOT IN (SELECT class_code FROM class_to_drug_manual) +; + +DELETE FROM class_to_drug -- inambiguous only for forms +WHERE class_code IN(SELECT class_code FROM interim) +; +INSERT INTO class_to_drug -- ingredients are partially inambiguous +(class_code, class_name, concept_id, concept_name, concept_class_id, "order") +SELECT class_code, class_name, c.concept_id, c.concept_name, c.concept_class_id,5 +FROM interim i +JOIN devv5.concept_ancestor ca on i.concept_id = ancestor_concept_id +JOIN concept c on descendant_concept_id = c.concept_id +WHERE c.concept_class_id = 'Clinical Drug Form' AND c.concept_name NOT LIKE '% / %' +; + +DELETE FROM class_to_drug +WHERE class_code IN(SELECT class_code FROM interim_2) +; +INSERT INTO class_to_drug -- ingredients are absolutely inambiguous +(class_code, class_name, concept_id, concept_name, concept_class_id, "order") +SELECT class_code, class_name, concept_id, c.concept_name, c.concept_class_id,6 +FROM interim_2 i +JOIN concept c USING (concept_id) +; + + +-- also adding those that don't have forms, but are unique +with ing AS ( +SELECT concept_id_2,concept_code_1, count(concept_id_2) AS cnt +FROM ( +SELECT DISTINCT substring(i.concept_code_1,'\w+'), r.concept_code_1, concept_id_2 +FROM relationship_to_concept r +JOIN internal_relationship_stage i on i.concept_code_2 = r.concept_code_1 +JOIN drug_concept_stage d on d.concept_code = i.concept_code_2 AND d.concept_class_id = 'Ingredient' + ) a GROUP BY concept_id_2,concept_code_1 ), +drug AS ( +SELECT DISTINCT substring(concept_code_1,'\w+') AS code,concept_code_2 +FROM internal_relationship_stage i +WHERE not exists (SELECT 1 FROM internal_relationship_stage i2 + WHERE i.concept_code_2=i2.concept_code_2 + AND substring(i.concept_code_1,'\w+')!=substring(i2.concept_code_1,'\w+')) +), + drug_name AS ( +SELECT DISTINCT class_code,class_name, concept_id_2 +FROM ing +JOIN drug on concept_code_2=concept_code_1 +JOIN class_drugs_scraper on code = class_code +WHERE cnt=1 AND class_name = concept_code_1 +and code NOT IN (SELECT class_code FROM class_to_drug) + ), +all_drug AS ( +SELECT class_code,class_name, concept_id,concept_name, concept_class_id, count(concept_id_2) OVER (PARTITION BY class_code) AS cnt +FROM drug_name +JOIN concept on concept_id_2 = concept_id +) +INSERT INTO class_to_drug +SELECT class_code, class_name, concept_id, concept_name,concept_class_id,'6' +FROM all_drug +WHERE cnt=1 +; + + -- CREATE TABLE for QA + DROP TABLE IF EXISTS class_to_drug_qa; + CREATE TABLE class_to_drug_qa + as + with first AS ( + SELECT cl.*,descendant_concept_id FROM class_to_drug cl + JOIN devv5.concept_ancestor ca on ca.ancestor_concept_id = cl.concept_id + WHERE "order"=1) + SELECT class_code, class_name, c.* + FROM first + JOIN concept c on descendant_concept_id = c.concept_id AND standard_concept='S' ; + + + INSERT INTO class_to_drug_qa + with second AS ( + SELECT cl.*,descendant_concept_id FROM class_to_drug cl + JOIN devv5.concept_ancestor ca on ca.ancestor_concept_id = cl.concept_id + WHERE "order"=2 AND descendant_concept_id NOT IN (SELECT concept_id FROM class_to_drug_qa) + ) + SELECT class_code, class_name, c.* + FROM second + JOIN concept c on descendant_concept_id = c.concept_id AND standard_concept='S' ; + + INSERT INTO class_to_drug_qa + with third AS ( + SELECT cl.*,descendant_concept_id FROM class_to_drug cl + JOIN devv5.concept_ancestor ca on ca.ancestor_concept_id = cl.concept_id + WHERE "order"=3 AND descendant_concept_id NOT IN (SELECT concept_id FROM class_to_drug_qa)) + SELECT class_code, class_name, c.* + FROM third + JOIN concept c on descendant_concept_id = c.concept_id AND standard_concept='S' ; + + INSERT INTO class_to_drug_qa + with forth AS ( + SELECT cl.*,descendant_concept_id FROM class_to_drug cl + JOIN devv5.concept_ancestor ca on ca.ancestor_concept_id = cl.concept_id + WHERE "order"=4 AND descendant_concept_id NOT IN (SELECT concept_id FROM class_to_drug_qa) + ) + SELECT class_code, class_name, c.* + FROM forth + JOIN concept c on descendant_concept_id = c.concept_id AND standard_concept='S' ; + + INSERT INTO class_to_drug_qa + SELECT DISTINCT class_code, class_name, c2.* + FROM class_to_drug cl + JOIN devv5.concept_ancestor ca on ca.ancestor_concept_id = cl.concept_id + JOIN concept c2 on c2.concept_id = descendant_concept_id + WHERE "order"=5 AND descendant_concept_id NOT IN (SELECT concept_id FROM class_to_drug_qa) + AND c2.concept_name NOT LIKE '% / %' -- to exclude multi-comp drugs that should go into comb. If its totally inamb ingredients, it will go AS 6 + ; + + INSERT INTO class_to_drug_qa + with sixth AS ( -- absolutely inamiguous + SELECT DISTINCT cl.*,descendant_concept_id + FROM class_to_drug cl + JOIN devv5.concept_ancestor ca on ca.ancestor_concept_id = cl.concept_id + WHERE "order"=6 + ) + SELECT class_code, class_name, c.* + FROM sixth + JOIN concept c on descendant_concept_id = c.concept_id AND standard_concept='S' + ; --5. Get tables for see what is missing --5.1 Check new class codes that should be worked out @@ -1056,20 +896,23 @@ SELECT * FROM class_drugs_scraper WHERE class_code NOT IN (SELECT concept_code FROM concept WHERE vocabulary_id = 'ATC'); + --5.2 Take a look at new standard ingredients that did not exist before the last ATC release SELECT * FROM concept -WHERE vocabulary_id in ('RxNorm', 'RxNorm Extension') +WHERE vocabulary_id IN('RxNorm', 'RxNorm Extension') AND concept_class_id = 'Ingredient' AND standard_concept = 'S' - AND valid_start_date > (select latest_update - from devv5.vocabulary_conversion - where vocabulary_id_v5 = 'ATC'); ---5.3 Take a look at the ATC codes that aren't covered in the release + AND valid_start_date > (SELECT latest_update + FROM devv5.vocabulary_conversion + WHERE vocabulary_id_v5 = 'ATC'); + +--5.3 Take a look at the ATC codes that aren't cOVERed INthe release SELECT * FROM class_drugs_scraper -WHERE class_code NOT IN (SELECT class_code FROM class_to_drug); - +WHERE class_code NOT IN (SELECT class_code FROM class_to_drug) +and length (class_code) =7 +; -- 6. Add ATC -- create temporary table atc_tmp_table @@ -1130,7 +973,7 @@ SELECT concept_name, concept_class_id, standard_concept, concept_code, - v.latest_update AS valid_start_date, + v.latest_UPDATE AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason FROM atc_tmp_table dv, @@ -1153,7 +996,7 @@ SELECT DISTINCT d.concept_code AS concept_code_1, 'VA Class to ATC eq' AS relationship_id, 'VA Class' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, - v.latest_update AS valid_start_date, + v.latest_UPDATE AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason FROM atc_tmp_table d @@ -1173,7 +1016,7 @@ SELECT DISTINCT d.concept_code AS concept_code_1, 'NDFRT to ATC eq' AS relationship_id, 'NDFRT' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, - v.latest_update AS valid_start_date, + v.latest_UPDATE AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason FROM atc_tmp_table d @@ -1198,7 +1041,7 @@ SELECT DISTINCT d.concept_code AS concept_code_1, 'NDFRT to ATC eq' AS relationship_id, 'NDFRT' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, - v.latest_update AS valid_start_date, + v.latest_UPDATE AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason FROM atc_tmp_table d @@ -1235,34 +1078,13 @@ WHERE d.vocabulary_id = 'SNOMED' UNION ALL --- add ATC to RxNorm -SELECT DISTINCT d.concept_code AS concept_code_1, - e.concept_code AS concept_code_2, - 'ATC - RxNorm' AS relationship_id, -- this is one to substitute "NDFRF has ing", is hierarchical AND defines ancestry. - 'ATC' AS vocabulary_id_1, - 'RxNorm' AS vocabulary_id_2, - v.latest_update AS valid_start_date, - TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM atc_tmp_table d -JOIN sources.rxnconso r ON r.rxcui = d.rxcui - AND r.code != 'NOCODE' -JOIN vocabulary v ON v.vocabulary_id = d.vocabulary_id -JOIN concept e ON r.rxcui = e.concept_code - AND e.vocabulary_id = 'RxNorm' - AND e.invalid_reason IS NULL -WHERE d.vocabulary_id = 'ATC' - AND d.concept_class_id = 'ATC 5th' -- there are some weird 4th level links, LIKE D11AC 'Medicated shampoos' to an RxNorm Dose Form - AND e.concept_class_id NOT IN ('Ingredient','Precise Ingredient') -UNION ALL - -- Hierarchy inside ATC SELECT uppr.concept_code AS concept_code_1, lowr.concept_code AS concept_code_2, 'Is a' AS relationship_id, 'ATC' AS vocabulary_id_1, 'ATC' AS vocabulary_id_2, - v.latest_update AS valid_start_date, + v.latest_UPDATE AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, NULL AS invalid_reason FROM concept_stage uppr, @@ -1299,15 +1121,15 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT DISTINCT cs.class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm' AS relationship_id, - CURRENT_DATE AS valid_start_date, +SELECT DISTINCT cs.class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm' AS relationship_id, + CURRENT_DATE AS valid_start_date, TO_DATE('20991231', 'yyyymmdd') AS valid_end_date, - NULL AS invalid_reason -FROM dev_atc.сlass_to_rx_descendant cs --manual source table + NULL AS invalid_reason +FROM class_to_drug cs --manual source table JOIN concept c ON c.concept_id = cs.concept_id WHERE NOT EXISTS ( SELECT 1 @@ -1317,9 +1139,92 @@ WHERE NOT EXISTS ( AND crs.concept_code_2 = c.concept_code AND crs.vocabulary_id_2 = c.vocabulary_id AND crs.relationship_id = 'ATC - RxNorm' - ); + ) +AND c.concept_class_id != 'Ingredient' -- 6 for unambiguous AND 1 for manual +; + --- 10. Add relationships to ingredients excluding multiple-ingredient combos +-- Primary unambiguous +INSERT INTO concept_relationship_stage (concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason) +SELECT DISTINCT substring(irs.concept_code_1, '\w+') AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM internal_relationship_stage irs + JOIN relationship_to_concept rtc on irs.concept_code_2 = rtc.concept_code_1 + JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' +WHERE rtc.concept_code_1 + NOT IN ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', + 'polyfructosans', 'triazoles', 'natural phospholipids', + 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', + 'antibiotics', 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', + 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', + 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', + 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', + 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', + 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', + 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', + 'barbiturates') + AND not exists(SELECT 1 + FROM ambiguous_class_ingredient_tst t + WHERE t.concept_code_2 = rtc.concept_code_1 + AND t.class_code = substring(irs.concept_code_1, '\w+') + AND flag = 'with') +; + +INSERT INTO concept_relationship_stage (concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason) +SELECT DISTINCT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'ing' + JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' +WHERE not exists(SELECT 1 + FROM ambiguous_class_ingredient_tst a2 + WHERE a.class_code = a2.class_code + AND a.rnk = a2.rnk + AND a.concept_code_2 != a2.concept_code_2) + AND rtc.concept_code_1 + NOT IN ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', + 'polyfructosans', 'triazoles', 'natural phospholipids', + 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', + 'antibiotics', 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', + 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', + 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', + 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', + 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', + 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', + 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', + 'barbiturates') + AND (class_code, concept_code,'ATC - RxNorm pr lat') not in + (SELECT concept_code_1,concept_code_2, relationship_id FROM concept_relationship_stage) + AND class_name != 'combinations' +; + +--manual table INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1330,32 +1235,191 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT DISTINCT cs.class_code AS concept_code_1, - c.concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - c.vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM dev_atc.сlass_to_rx_descendant cs -JOIN devv5.concept_ancestor ca ON ca.descendant_concept_id = cs.concept_id -JOIN concept c ON c.concept_id = ca.ancestor_concept_id - AND c.concept_class_id = 'Ingredient' - AND c.vocabulary_id LIKE 'RxNorm%' -WHERE NOT cs.class_name ~ 'combination|agents|drugs|supplements|corticosteroids|compounds|sulfonylureas|preparations|thiazides|antacid|antiinfectives|calcium$|potassium$|sodium$|antiseptics|antibiotics|mydriatics|psycholeptic|other|diuretic|nitrates|analgesics' - AND c.concept_name NOT IN ('Inert Ingredients') -- a component of contraceptive packs - AND NOT EXISTS ( - SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = cs.class_code - AND crs.vocabulary_id_1 = 'ATC' - AND crs.concept_code_2 = c.concept_code - AND crs.vocabulary_id_2 = c.vocabulary_id - AND crs.relationship_id = 'ATC - RxNorm' - ); +SELECT class_code AS concept_code_1, + concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM class_to_drug_manual_ing +WHERE flag = 0 +UNION +SELECT class_code AS concept_code_1, + concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM class_to_drug_manual_ing +WHERE flag = 1 + AND class_name != 'combinations' + AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) + AND class_code NOT IN ('J06AA03');--snake venom antiserum + + +-- Secondary unambiguous +INSERT INTO concept_relationship_stage + (concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT DISTINCT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'with' + JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' +WHERE not exists(SELECT 1 + FROM ambiguous_class_ingredient_tst a2 + WHERE a.class_code = a2.class_code + AND a.rnk = a2.rnk + AND a.concept_code_2 != a2.concept_code_2) + AND rtc.concept_code_1 + NOT IN ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', + 'polyfructosans', 'triazoles', 'natural phospholipids', + 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', + 'antibiotics', 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', + 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', + 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', + 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', + 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', + 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', + 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', + 'barbiturates') + AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) + AND class_name != 'combinations' + AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage); + +--manual table +INSERT INTO concept_relationship_stage + (concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT DISTINCT class_code AS concept_code_1, + concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec lat' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM class_to_drug_manual_ing +WHERE flag > 1 + AND class_name != 'combinations' + AND (class_code, concept_code) NOT IN ( + SELECT concept_code_1,concept_code_2 + FROM concept_relationship_stage); + +-- Primary ambiguous +INSERT INTO concept_relationship_stage + (concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'ing' + JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' +WHERE (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) + AND rtc.concept_code_1 + IN('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', + 'polyfructosans', 'triazoles','natural phospholipids', + 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', 'antibiotics', + 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', + 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', + 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', + 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', + 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', + 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', + 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', 'barbiturates') +UNION +SELECT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 AND flag = 'ing' + JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' +WHERE exists(SELECT 1 + FROM ambiguous_class_ingredient_tst a2 + WHERE a.class_code = a2.class_code + AND a.rnk = a2.rnk + AND a.concept_code_2 != a2.concept_code_2) +; + +INSERT INTO concept_relationship_stage + (concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT DISTINCT substring(irs.concept_code_1, '\w+') AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM internal_relationship_stage irs + JOIN relationship_to_concept rtc ON irs.concept_code_2 = rtc.concept_code_1 + JOIN concept c ON concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' +WHERE rtc.concept_code_1 + IN('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', + 'polyfructosans', 'triazoles','natural phospholipids', + 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', 'antibiotics', + 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', + 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', + 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', + 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', + 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', + 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', + 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', 'barbiturates') + AND (substring(irs.concept_code_1, '\w+'), concept_code) not in + (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) + AND substring(irs.concept_code_1, '\w+') NOT IN (SELECT class_code FROM ambiguous_class_ingredient_tst) +; --- 11. Add relationships to ingredients for combo drugs where possible INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1366,62 +1430,21 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -SELECT class_code AS concept_code_1, - concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM ( - SELECT am.class_code, - c.concept_code, - c.vocabulary_id - FROM dev_atc.ambiguous_class_ingredient am - JOIN dev_atc.relationship_to_concept rtc ON rtc.concept_code_1 = am.ing - JOIN concept c ON rtc.concept_id_2 = c.concept_id - WHERE am.flag = 'ing' - AND NOT EXISTS ( - SELECT 1 - FROM dev_atc.ambiguous_class_ingredient am2 - WHERE am.class_code = am2.class_code - AND am.ing = am2.ing - AND rtc.precedence > 1 - ) - - UNION - - SELECT a.class_code, - c.concept_code, - c.vocabulary_id - FROM dev_atc.class_1_comb a - LEFT JOIN dev_atc.reference r ON a.class_code = r.class_code - JOIN dev_atc.internal_relationship_stage i ON coalesce(concept_code, a.class_code) = i.concept_code_1 - JOIN dev_atc.drug_concept_stage d ON d.concept_code = i.concept_code_2 - AND d.concept_class_id = 'Ingredient' - JOIN dev_atc.relationship_to_concept rtc ON rtc.concept_code_1 = d.concept_code - JOIN concept c ON rtc.concept_id_2 = c.concept_id - WHERE a.class_name ~ 'comb| and ' - AND NOT EXISTS ( - SELECT 1 - FROM dev_atc.relationship_to_concept rtc2 - WHERE rtc.concept_code_1 = rtc2.concept_code_1 - AND rtc2.precedence > 1 - ) - ) atc -WHERE NOT EXISTS ( - SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = atc.class_code - AND crs.vocabulary_id_1 = 'ATC' - AND crs.concept_code_2 = atc.concept_code - AND crs.vocabulary_id_2 = atc.vocabulary_id - AND crs.relationship_id = 'ATC - RxNorm' - ); - - --- 12. Add ingredients from those ATC drugs that didn't match to RxNorm concepts +SELECT class_code AS concept_code_1, + concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM class_to_drug_manual_ing +WHERE flag = 1 + AND class_name != 'combinations' + AND (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) + AND class_code IN ('J06AA03');--snake venom antiserum + +-- combinations, unambiguous, all secondary INSERT INTO concept_relationship_stage ( concept_code_1, concept_code_2, @@ -1432,189 +1455,260 @@ INSERT INTO concept_relationship_stage ( valid_end_date, invalid_reason ) -WITH corpus AS ( - SELECT DISTINCT class_code, - class_name, - i.concept_code_2, - rtc.concept_code_1, - c.concept_code, - c.vocabulary_id, - c.concept_name - FROM internal_relationship_stage i - JOIN class_drugs_scraper cds ON substring(concept_code_1, '\w+') = class_code - LEFT JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 - LEFT JOIN drug_concept_stage dcs ON dcs.concept_code = i.concept_code_2 AND dcs.concept_class_id = 'Ingredient' - JOIN concept c ON c.concept_id = rtc.concept_id_2 AND c.concept_class_id = 'Ingredient' - WHERE class_code NOT IN ( - SELECT class_code - FROM class_to_rx_descendant - ) - AND NOT EXISTS( - SELECT 1 - FROM relationship_to_concept rtc2 - WHERE rtc2.concept_code_1 = rtc.concept_code_1 - AND precedence > 1 - )) -SELECT class_code AS concept_code_1, - concept_code AS concept_code_2, - 'ATC' AS vocabulary_id_1, - vocabulary_id AS vocabulary_id_2, - 'ATC - RxNorm' AS relationship_id, - CURRENT_DATE AS valid_start_date, - TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, - NULL AS invalid_reason -FROM corpus c -WHERE NOT EXISTS ( - SELECT 1 - FROM corpus c2 - WHERE c.class_code = c2.class_code - AND c2.concept_code_1 IS NULL +SELECT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm pr up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM (SELECT class_code,c.concept_code,c.vocabulary_id + FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc on a.concept_code_2 = rtc.concept_code_1 + JOIN concept c on concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' + WHERE class_name = 'combinations' + UNION + SELECT class_code,concept_code, vocabulary_id + FROM class_to_drug_manual_ing + WHERE class_name = 'combinations' + ) c +WHERE (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage); + + +-- Secondary ambiguous +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT DISTINCT class_code AS concept_code_1, + c.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + c.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM ambiguous_class_ingredient_tst a + JOIN relationship_to_concept rtc ON a.concept_code_2 = rtc.concept_code_1 + JOIN concept c ON concept_id_2 = c.concept_id AND c.concept_class_id = 'Ingredient' +WHERE (exists(SELECT 1 + FROM ambiguous_class_ingredient_tst a2 + WHERE a.class_code = a2.class_code + AND a.rnk = a2.rnk + AND a.concept_code_2 != a2.concept_code_2) + OR rtc.concept_code_1 + IN + ('adrenergics', 'analgesics', 'antispasmodics', 'antibacterials', 'antiflatulents', 'imidazoles', 'minerals', + 'polyfructosans', 'triazoles', 'natural phospholipids', + 'lactic acid producing organisms', 'antiflatulents', 'beta-lactamase inhibitor', 'sulfonylureas', 'antibiotics', + 'antiinfectives', 'antiseptics', 'artificial tears', 'contact laxatives', 'corticosteroids', + 'ordinary salt combinations', 'imidazoles/triazoles', 'cough suppressants', 'diuretics', + 'drugs for obstructive airway diseases', 'expectorants', 'belladonna alkaloids', 'mucolytics', 'mydriatics', + 'non-opioid analgesics', 'organic nitrates', 'potassium-sparing agents', 'proton pump inhibitors', + 'psycholeptics', 'thiazides', 'snake venom antiserum', 'fat emulsions', + 'amino acids', 'acid preparations', 'sulfur compounds', 'stramoni preparations', 'protamines', 'penicillins', + 'comt inhibitor', 'cannabinoids', 'decarboxylase inhibitor', 'edetates', 'barbiturates') ) -AND NOT EXISTS ( - SELECT 1 - FROM concept_relationship_stage crs - WHERE crs.concept_code_1 = c.class_code - AND crs.vocabulary_id_1 = 'ATC' - AND crs.concept_code_2 = c.concept_code - AND crs.vocabulary_id_2 = c.vocabulary_id - AND crs.relationship_id = 'ATC - RxNorm' - ) -; - --- 13. Add manual relationships -DO $_$ -BEGIN - PERFORM VOCABULARY_PACK.ProcessManualRelationships(); -END $_$; + AND flag = 'with' + AND class_name != 'combinations' + and (class_code, concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) +; --- 14. Remove ATC's duplicates (AVOF-322) --- diphtheria immunoglobulin -DELETE FROM concept_relationship_stage WHERE concept_code_1 = 'J06BB10' AND concept_code_2 = '3510' AND relationship_id = 'ATC - RxNorm'; --- hydroquinine -DELETE FROM concept_relationship_stage WHERE concept_code_1 = 'M09AA01' AND concept_code_2 = '27220' AND relationship_id = 'ATC - RxNorm'; +-- add 'x,combinations' +WITH main_ing AS ( + SELECT class_code, concept_id_2 + FROM class_to_drug c + JOIN internal_relationship_stage i ON c.class_code = substring(i.concept_code_1, '\w+') + JOIN relationship_to_concept rtc ON rtc.concept_code_1 = i.concept_code_2 + WHERE class_name ~ '(, combinations)|(in combination with other drugs)' + AND NOT class_name ~ 'with|and thiazides|and other diuretics' -- gives errors INC07BB52 AND C07CB53 if removed +) +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT DISTINCT class_code AS concept_code_1, + cc.concept_code AS concept_code_2, + 'ATC' AS vocabulary_id_1, + cc.vocabulary_id AS vocabulary_id_2, + 'ATC - RxNorm sec up' AS relationship_id, + CURRENT_DATE AS valid_start_date, + TO_DATE('20991231', 'YYYYMMDD') AS valid_end_date, + NULL AS invalid_reason +FROM class_to_drug c + JOIN devv5.concept_ancestor ON descendant_concept_id = c.concept_id + JOIN concept cc ON cc.concept_id = ancestor_concept_id AND cc.standard_concept = 'S' + AND cc.concept_class_id = 'Ingredient' AND cc.vocabulary_id LIKE 'Rx%' +WHERE class_name ~ '(, combinations)|(in combination with other drugs)' + AND NOT class_name ~ 'with|and thiazides|and other diuretics' + AND (c.class_code, cc.concept_id) NOT IN (SELECT class_code, concept_id_2 FROM main_ing) + AND (class_code, cc.concept_code) NOT IN (SELECT concept_code_1,concept_code_2 FROM concept_relationship_stage) +; --- 15. Deprecate relationships between multi ingredient drugs and a single ATC 5th, because it should have either an ATC for each ingredient or an ATC that is a combination of them --- 15.1. Create temporary table drug_strength_ext (same code AS IN concept_ancestor, but we exclude ds for ingredients (because we use count(*)>1 AND ds for ingredients HAVING COUNT(*)=1) AND only for RxNorm) -DROP TABLE IF EXISTS drug_strength_ext; -CREATE UNLOGGED TABLE drug_strength_ext AS -SELECT * + + +-- XXX 8.1 temporary deprecate old rels +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT c.concept_code AS concept_code_1, + c.vocabulary_id AS vocabulary_id_1, + cc.concept_code AS concept_code_2, + cc.vocabulary_id AS vocabulary_id_2, + relationship_id AS relationship_id, + 'D' AS invalid_reason, + cr.valid_start_date AS valid_start_date, + current_date AS valid_end_date +FROM concept_relationship cr + JOIN concept c on concept_id_1 = c.concept_id + JOIN concept cc on concept_id_2 = cc.concept_id +WHERE c.vocabulary_id = 'ATC' + AND cc.vocabulary_id LIKE 'RxNorm%' + AND relationship_id IN ('ATC - RxNorm', 'ATC - RxNorm name') -- 'Maps to', 'Drug class of drug', + AND cr.invalid_reason IS NULL + and (c.concept_code, relationship_id, cc.concept_code) NOT IN + (SELECT concept_code_1,relationship_id, concept_code_2 FROM concept_relationship_stage) +; + +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT c.concept_code, + c.vocabulary_id, + cc.concept_code, + cc.vocabulary_id, + relationship_id, + cr.invalid_reason, + cr.valid_start_date, + cr.valid_end_date +FROM concept_relationship cr + JOIN concept c on c.concept_id = concept_id_1 + JOIN concept cc on cc.concept_id = concept_id_2 +WHERE cr.invalid_reason IS NULL + AND cr.relationship_id = 'Maps to' + AND c.vocabulary_id = 'ATC' + AND cc.vocabulary_id IN('RxNorm', 'RxNorm Extension', 'CVX') + AND (c.concept_code, cc.concept_code) IN(SELECT concept_code_1,concept_code_2 + FROM concept_relationship_stage + WHERE relationship_id = 'ATC - RxNorm pr lat'); + +-- primary lateral always go inside, >25 - calcium, 23-24 J07% +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT concept_code_1, + vocabulary_id_1, + concept_code_2, + vocabulary_id_2, + 'Maps to', + invalid_reason, + valid_start_date, + valid_end_date FROM ( - WITH ingredient_unit AS ( - SELECT DISTINCT - -- pick the most common unit for an ingredient. If there is a draw, pick always the same by sorting by unit_concept_id - ingredient_concept_code, - vocabulary_id, - FIRST_VALUE(unit_concept_id) OVER ( - PARTITION BY ingredient_concept_code ORDER BY cnt DESC, - unit_concept_id - ) AS unit_concept_id - FROM ( - -- sum the counts coming FROM amount AND numerator - SELECT ingredient_concept_code, - vocabulary_id, - unit_concept_id, - SUM(cnt) AS cnt - FROM ( - -- count ingredients, their units AND the frequency - SELECT c2.concept_code AS ingredient_concept_code, - c2.vocabulary_id, - ds.amount_unit_concept_id AS unit_concept_id, - COUNT(*) AS cnt - FROM drug_strength ds - JOIN concept c1 ON c1.concept_id = ds.drug_concept_id - AND c1.vocabulary_id = 'RxNorm' - JOIN concept c2 ON c2.concept_id = ds.ingredient_concept_id - AND c2.vocabulary_id = 'RxNorm' - WHERE ds.amount_value <> 0 - GROUP BY c2.concept_code, - c2.vocabulary_id, - ds.amount_unit_concept_id - - UNION - - SELECT c2.concept_code AS ingredient_concept_code, - c2.vocabulary_id, - ds.numerator_unit_concept_id AS unit_concept_id, - COUNT(*) AS cnt - FROM drug_strength ds - JOIN concept c1 ON c1.concept_id = ds.drug_concept_id - AND c1.vocabulary_id = 'RxNorm' - JOIN concept c2 ON c2.concept_id = ds.ingredient_concept_id - AND c2.vocabulary_id = 'RxNorm' - WHERE ds.numerator_value <> 0 - GROUP BY c2.concept_code, - c2.vocabulary_id, - ds.numerator_unit_concept_id - ) AS s0 - GROUP BY ingredient_concept_code, - vocabulary_id, - unit_concept_id - ) AS s1 - ) - -- Create drug_strength for drug forms - SELECT de.concept_code AS drug_concept_code, - an.concept_code AS ingredient_concept_code - FROM concept an - JOIN devv5.concept_ancestor ca ON ca.ancestor_concept_id = an.concept_id - JOIN concept de ON de.concept_id = ca.descendant_concept_id - JOIN ingredient_unit iu ON iu.ingredient_concept_code = an.concept_code - AND iu.vocabulary_id = an.vocabulary_id - WHERE an.vocabulary_id = 'RxNorm' - AND an.concept_class_id = 'Ingredient' - AND de.vocabulary_id = 'RxNorm' - AND de.concept_class_id IN ( - 'Clinical Drug Form', - 'Branded Drug Form' - ) - ) AS s2; +SELECT *, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt +FROM concept_relationship_stage +WHERE relationship_id IN('ATC - RxNorm pr lat') -- 'Maps to', 'Drug class of drug', +and invalid_reason IS NULL +and (concept_code_1, vocabulary_id_1, 'Maps to', concept_code_2,vocabulary_id_2) + NOT IN (SELECT concept_code_1, vocabulary_id_1, relationship_id, concept_code_2,vocabulary_id_2 FROM concept_relationship_stage) + ) a +WHERE cnt<25 +; --- 15.2. Do deprecation -DELETE +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT concept_code_1, + vocabulary_id_1, + concept_code_2, + vocabulary_id_2, + 'Maps to', + invalid_reason, + valid_start_date, + valid_end_date +FROM ( +SELECT *, count(concept_code_2) OVER (PARTITION BY concept_code_1) AS cnt FROM concept_relationship_stage -WHERE ctid IN ( - SELECT drug2atc.row_id - FROM ( - SELECT drug_concept_code - FROM ( - SELECT c1.concept_code AS drug_concept_code, - c2.concept_code - FROM drug_strength ds - JOIN concept c1 ON c1.concept_id = ds.drug_concept_id - AND c1.vocabulary_id = 'RxNorm' - JOIN concept c2 ON c2.concept_id = ds.ingredient_concept_id - - UNION - - SELECT drug_concept_code, - ingredient_concept_code - FROM drug_strength_ext - ) AS s0 - GROUP BY drug_concept_code - HAVING count(*) > 1 - ) all_drugs - JOIN ( - SELECT * - FROM ( - SELECT crs.ctid row_id, - crs.concept_code_2, - count(*) OVER (PARTITION BY crs.concept_code_2) cnt_atc - FROM concept_relationship_stage crs - JOIN concept_stage cs ON cs.concept_code = crs.concept_code_1 - AND cs.vocabulary_id = crs.vocabulary_id_1 - AND cs.vocabulary_id = 'ATC' - AND cs.concept_class_id = 'ATC 5th' - AND NOT cs.concept_name ~ 'preparations|virus|antigen|-|/|organisms|insulin|etc\.|influenza|human menopausal gonadotrophin|combination|amino acids|electrolytes| AND |excl\.| with |others|various' - JOIN concept c ON c.concept_code = crs.concept_code_2 - AND c.vocabulary_id = crs.vocabulary_id_2 - AND c.vocabulary_id = 'RxNorm' - WHERE crs.relationship_id = 'ATC - RxNorm' - AND crs.invalid_reason IS NULL - ) AS s1 - WHERE cnt_atc = 1 - ) drug2atc ON drug2atc.concept_code_2 = all_drugs.drug_concept_code - ); - --- 16. Add synonyms to concept_synonym stage for each of the rxcui/code combinations IN atc_tmp_table +WHERE relationship_id IN('ATC - RxNorm sec lat') -- 'Maps to', 'Drug class of drug', +and invalid_reason IS NULL +and (concept_code_1, vocabulary_id_1, 'Maps to', concept_code_2,vocabulary_id_2) + NOT IN (SELECT concept_code_1, vocabulary_id_1, relationship_id, concept_code_2,vocabulary_id_2 FROM concept_relationship_stage) + ) a +WHERE cnt<10 +; + +-- add deprecated Maps to +INSERT INTO concept_relationship_stage ( + concept_code_1, + concept_code_2, + vocabulary_id_1, + vocabulary_id_2, + relationship_id, + valid_start_date, + valid_end_date, + invalid_reason + ) +SELECT c.concept_code, + c.vocabulary_id, + cc.concept_code, + cc.vocabulary_id, + relationship_id, + 'D', + cr.valid_start_date, + current_date +FROM concept_relationship cr + JOIN concept c on c.concept_id = concept_id_1 + JOIN concept cc on cc.concept_id = concept_id_2 +WHERE cr.invalid_reason IS NULL + AND cr.relationship_id = 'Maps to' + AND c.vocabulary_id = 'ATC' AND c.invalid_reason IS NULL + AND cc.vocabulary_id IN('RxNorm', 'RxNorm Extension', 'CVX') + AND (c.concept_code, cr.relationship_id, cc.concept_code) NOT IN (SELECT concept_code_1,relationship_id, concept_code_2 + FROM concept_relationship_stage) + ; + +-- 16. Add synonyms to concept_synonym stage for each of the rxcui/code combinations in atc_tmp_table INSERT INTO concept_synonym_stage ( synonym_concept_code, synonym_name, @@ -1630,11 +1724,74 @@ JOIN sources.rxnconso r ON dv.code = r.code AND dv.rxcui = r.rxcui AND r.code != 'NOCODE' AND r.lat = 'ENG'; - --- 17. Deprecate RxNorm 'Maps to' -DELETE -FROM concept_relationship_stage -WHERE relationship_id IN ('Maps to','Mapped from'); + + +-- 16.2 update concept names for combos +UPDATE concept_stage c +set concept_name = name +FROM (SELECT c2.concept_name||' - '||c.concept_name AS name, c.concept_code +FROM concept_stage c +JOIN concept_relationship_stage cr on cr.concept_code_1 = c.concept_code +JOIN concept_stage c2 on c2.concept_code = cr.concept_code_2 AND relationship_id = 'Is a' +WHERE c.concept_name IN('various', 'combinations','others')) a +WHERE (c.concept_code = a.concept_code); + +-- 16.3 add forms to names +DROP TABLE IF EXISTS new_name_form; +CREATE TABLE new_name_form +AS +WITH name AS ( + SELECT class_code, + class_name, + CASE + WHEN adm_r = 'O' THEN 'oral' + WHEN adm_r = 'Chewing gum' THEN 'chewing gum' + WHEN adm_r LIKE '%Inhal%' THEN 'inhalant' + WHEN adm_r = 'Instill.sol.' THEN 'instillation solution' + WHEN adm_r = 'N' THEN 'nasal' + WHEN adm_r = 'P' THEN 'parenteral' + WHEN adm_r = 'R' THEN 'rectal' + WHEN adm_r = 'SL' THEN 'sublingual' + WHEN adm_r LIKE 'TD%' THEN 'transdermal' + WHEN adm_r = 'V' THEN 'vaginal' + WHEN adm_r IN('implant', 'intravesical', 'ointment', 'oral aerosol', 'urethral') THEN adm_r + WHEN adm_r = 's.c. implant' THEN 'implant' + ELSE null END AS form + FROM class_drugs_scraper + WHERE adm_r IS NOT NULL + UNION + SELECT class_code, + class_name, + CASE + WHEN class_code LIKE 'A01AB%' THEN 'local oral' + WHEN concept_code LIKE '%Oral%' THEN 'oral' + WHEN concept_code LIKE '%Rectal%' THEN 'rectal' + WHEN concept_code LIKE '%Vaginal%' THEN 'vaginal' + WHEN concept_code LIKE '%Injectable%' THEN 'parenteral' + WHEN concept_code LIKE '%Inhal%' THEN 'inhalant' + WHEN concept_code LIKE '%Nasal%' THEN 'nasal' + WHEN concept_code LIKE '%Ophthalmic%' THEN 'ophthalmic' + WHEN concept_code LIKE '%Topical%' THEN 'topical' + WHEN concept_code LIKE '%Ophthalmic%' THEN 'ophthalmic' + ELSE null END AS form + FROM reference + JOIN class_drugs_scraper USING (class_code) + WHERE class_code != concept_code + AND adm_r IS NULL +), +forms AS ( +SELECT class_code,class_name, + regexp_replace(string_agg(form, ', ' ORDER BY class_code, form),'oral, parenteral','systemic') AS form_list +FROM name +WHERE form IS NOT NULL +GROUP BY class_code,class_name) +SELECT class_code, class_name||'; '||form_list AS new_name + FROM forms; + +UPDATE concept_stage +set concept_name = new_name +FROM (SELECT class_code, new_name FROM new_name_form) a +WHERE (concept_code = class_code); -- 18. Working with replacement mappings DO $_$ @@ -1642,13 +1799,13 @@ BEGIN PERFORM VOCABULARY_PACK.CheckReplacementMappings(); END $_$; --- 19. Add mapping from deprecated to fresh concepts +-- 20. Add mapping FROM deprecated to fresh concepts DO $_$ BEGIN PERFORM VOCABULARY_PACK.AddFreshMAPSTO(); END $_$; --- 20. Deprecate 'Maps to' mappings to deprecated and upgraded concepts +-- 19. Deprecate 'Maps to' mappings to deprecated AND upgraded concepts DO $_$ BEGIN PERFORM VOCABULARY_PACK.DeprecateWrongMAPSTO(); @@ -1660,30 +1817,5 @@ BEGIN PERFORM VOCABULARY_PACK.DELETEAmbiguousMAPSTO(); END $_$; --- 22. DELETE mappings between concepts that are not represented at the "latest_update" at this moment (e.g. SNOMED <-> RxNorm, but currently we are updating ATC) ---This is because we have SNOMED <-> ATC IN concept_relationship_stage, but AddFreshMAPSTO adds SNOMED <-> RxNorm FROM concept_relationship -DELETE -FROM concept_relationship_stage crs_o -WHERE ( - crs_o.concept_code_1, - crs_o.vocabulary_id_1, - crs_o.concept_code_2, - crs_o.vocabulary_id_2 - ) IN ( - SELECT crs.concept_code_1, - crs.vocabulary_id_1, - crs.concept_code_2, - crs.vocabulary_id_2 - FROM concept_relationship_stage crs - LEFT JOIN vocabulary v1 ON v1.vocabulary_id = crs.vocabulary_id_1 - AND v1.latest_update IS NOT NULL - LEFT JOIN vocabulary v2 ON v2.vocabulary_id = crs.vocabulary_id_2 - AND v2.latest_update IS NOT NULL - WHERE coalesce(v1.latest_update, v2.latest_update) IS NULL - ); - --- 23. Clean up -DROP TABLE atc_tmp_table; -DROP TABLE drug_strength_ext; - --- At the end, the three tables concept_stage, concept_relationship_stage and concept_synonym_stage should be ready to be fed into the generic_update.sql script +DELETE FROM concept_relationship_stage +WHERE vocabulary_id_1='SNOMED' AND vocabulary_id_2 = 'RxNorm';