From 0987027caec4eefe0fc0a9cb4845c2a234ffef5a Mon Sep 17 00:00:00 2001 From: Andy Boughton Date: Wed, 1 Dec 2021 17:30:57 -0500 Subject: [PATCH] Update PheWeb for changes in LZjs 0.14.0 Simplifies the data retrieval code (zomg), and contains minor visual improvements for use of figures in publications: mostly larger font sizes and more concise legend --- pheweb/conf.py | 2 +- pheweb/serve/server_utils.py | 4 + pheweb/serve/static/region.css | 2 +- pheweb/serve/static/region.js | 141 +++++++++++++-------------------- pheweb/serve/static/variant.js | 48 +++++------ 5 files changed, 79 insertions(+), 118 deletions(-) diff --git a/pheweb/conf.py b/pheweb/conf.py index 2c030e738..9c5ed81fd 100644 --- a/pheweb/conf.py +++ b/pheweb/conf.py @@ -164,7 +164,7 @@ def get_pheno_correlations_pvalue_threshold() -> float: return _get_config_float ## Serving config -def get_lzjs_version() -> str: return _get_config_str('lzjs_version', '0.13.0') +def get_lzjs_version() -> str: return _get_config_str('lzjs_version', '0.14.0-beta.2') def should_allow_variant_json_cors() -> bool: return _get_config_bool('allow_variant_json_cors', True) def get_urlprefix() -> str: return _get_config_str('urlprefix', '').rstrip('/') def get_custom_templates_dir() -> Optional[str]: diff --git a/pheweb/serve/server_utils.py b/pheweb/serve/server_utils.py index 8c9aa47c4..8a0f7b3fe 100644 --- a/pheweb/serve/server_utils.py +++ b/pheweb/serve/server_utils.py @@ -1,3 +1,4 @@ +import math from flask import url_for, Response, redirect @@ -37,7 +38,10 @@ def get_pheno_region(phenocode:str, chrom:str, pos_start:int, pos_end:int) -> di _Get_Pheno_Region._rename(v, 'chrom', 'chr') _Get_Pheno_Region._rename(v, 'pos', 'position') _Get_Pheno_Region._rename(v, 'rsids', 'rsid') + # TODO: Old PheWeb sends "pvalue", but new LocusZoom is trying to encourage use of log_pvalue (more resistant to underflow for really significant hits) + # We will send both fields for now, but should consolidate this in future. _Get_Pheno_Region._rename(v, 'pval', 'pvalue') + v['log_pvalue'] = -math.log10(v['pvalue']) variants.append(v) df = _Get_Pheno_Region._dataframify(variants) diff --git a/pheweb/serve/static/region.css b/pheweb/serve/static/region.css index f792f7cf8..ec662009c 100644 --- a/pheweb/serve/static/region.css +++ b/pheweb/serve/static/region.css @@ -8,7 +8,7 @@ float: none !important; } /* -TODO: merge toolbar-component-centering into LZ. (should it be the default?) +TODO: merge toolbar-widget-centering into LZ. (should it be the default?) - if not, make it explicit. it's bad that I'm just centering anything with `.lz-toolbar-group-*`. TODO: stop text-align:center from cascading downward through descendants */ diff --git a/pheweb/serve/static/region.js b/pheweb/serve/static/region.js index 15ae28e9a..2d563eaa1 100644 --- a/pheweb/serve/static/region.js +++ b/pheweb/serve/static/region.js @@ -1,46 +1,26 @@ 'use strict'; LocusZoom.Adapters.extend("AssociationLZ", "AssociationPheWeb", { - getURL: function (state, chain, fields) { - return this.url + "results/?filter=chromosome in '" + state.chr + "'" + - " and position ge " + state.start + - " and position le " + state.end; - }, - // Although the layout fields array is useful for specifying transforms, this source will magically re-add - // any data that was not explicitly requested - extractFields: function(data, fields, outnames, trans) { - // The field "all" has a special meaning, and only exists to trigger a request to this source. - // We're not actually trying to request a field by that name. - var has_all = fields.indexOf("all"); - if (has_all !== -1) { - fields.splice(has_all, 1); - outnames.splice(has_all, 1); - trans.splice(has_all, 1); - } - // Find all fields that have not been requested (sans transforms), and add them back to the fields array - if (data.length) { - var fieldnames = Object.keys(data[0]); - var ns = this.source_id + ":"; // ensure that namespacing is applied to the fields - fieldnames.forEach(function(item) { - var ref = fields.indexOf(item); - if (ref === -1 || trans[ref]) { - fields.push(item); - outnames.push(ns + item); - trans.push(null); - } - }); - } - return LocusZoom.Adapters.get('AssociationLZ').prototype.extractFields.call(this, data, fields, outnames, trans); + _getURL: function (request_options) { + const { chr, start, end } = request_options; + return `${this._url}results/?filter=chromosome in '${chr}' and position ge ${start} and position le ${end}`; }, - normalizeResponse(data) { + _normalizeResponse(response_text) { // The PheWeb region API has a fun quirk where if there is no data, there are also no keys // (eg data = {} instead of {assoc:[]} etc. Explicitly detect and handle the edge case in PheWeb; // we won't handle this in LZ core because we don't want squishy-blob API schemas to catch on. - if (!Object.keys(data).length) { + const data = JSON.parse(response_text); + if (!Object.keys(data.data).length) { return []; } - return LocusZoom.Adapters.get('AssociationLZ').prototype.normalizeResponse.call(this, data); + return LocusZoom.Adapters.get('AssociationLZ').prototype._normalizeResponse.call(this, data); + }, + + _annotateRecords(records) { + // The LD Adapter expects to see a field named "variant" with variant specifier. PheWeb API calls this "ID". + records.forEach((item) => item.variant = item.id); + return records; } }); @@ -58,12 +38,10 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { var remoteBase = "https://portaldev.sph.umich.edu/api/v1/"; var data_sources = new LocusZoom.DataSources() .add("assoc", ["AssociationPheWeb", {url: localBase }]) - .add("catalog", ["GwasCatalogLZ", {url: remoteBase + 'annotation/gwascatalog/results/', params: { build: "GRCh"+window.model.grch_build_number }}]) - .add("ld", ["LDServer", { url: "https://portaldev.sph.umich.edu/ld/", - params: { source: '1000G', build: 'GRCh'+window.model.grch_build_number, population: 'ALL' } - }]) - .add("gene", ["GeneLZ", { url: remoteBase + "annotation/genes/", params: {build: 'GRCh'+window.model.grch_build_number} }]) - .add("recomb", ["RecombLZ", { url: remoteBase + "annotation/recomb/results/", params: {build:'GRCh'+window.model.grch_build_number} }]); + .add("catalog", ["GwasCatalogLZ", {url: remoteBase + 'annotation/gwascatalog/results/', build: "GRCh"+window.model.grch_build_number }]) + .add("ld", ["LDServer", { url: "https://portaldev.sph.umich.edu/ld/", source: '1000G', build: 'GRCh'+window.model.grch_build_number, population: 'ALL' }]) + .add("gene", ["GeneLZ", { url: remoteBase + "annotation/genes/", build: 'GRCh'+window.model.grch_build_number }]) + .add("recomb", ["RecombLZ", { url: remoteBase + "annotation/recomb/results/", build:'GRCh'+window.model.grch_build_number }]); LocusZoom.TransformationFunctions.add("neglog10_or_323", function(x) { if (x === 0) return 323; @@ -105,9 +83,7 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { // Define the layout, then fetch it via the LZ machinery responsible for namespacing var layout = LocusZoom.Layouts.get("plot", "association_catalog", { - unnamespaced: true, width: 800, - // height: 550, responsive_resize: true, max_region_scale: 500e3, toolbar: { @@ -164,13 +140,11 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { panels: [ function() { var base = LocusZoom.Layouts.get("panel", "annotation_catalog", { - unnamespaced: true, height: 52, min_height: 52, margin: { top: 30, bottom: 13 }, toolbar: { widgets: [] }, axes: { - // FIXME: Can be removed after 0.13.1 bugfix release (render: false was missing) - x: { render: false, extent: 'state' } + x: { extent: 'state' } }, title: { text: 'Hits in GWAS Catalog', @@ -179,20 +153,15 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { }, }); var anno_layer = base.data_layers[0]; - anno_layer.id_field = "{{namespace[assoc]}}id"; - anno_layer.fields = [ // Tell annotation track the field names as used by PheWeb - "{{namespace[assoc]}}id", - "{{namespace[assoc]}}chr", "{{namespace[assoc]}}position", - "{{namespace[catalog]}}variant", "{{namespace[catalog]}}rsid", "{{namespace[catalog]}}trait", "{{namespace[catalog]}}log_pvalue" - ]; + anno_layer.id_field = "assoc:id"; anno_layer.hit_area_width = 50; return base; }(), function() { - // FIXME: The only customization here is to make the legend button green and hide the "move panel" buttons; displayn options doesn't need to be copy-pasted + // FIXME: The only customization here is to make the legend button green and hide the "move panel" buttons; display options doesn't need to be copy-pasted var base = LocusZoom.Layouts.get("panel", "association_catalog", { - unnamespaced: true, - height: 200, min_height: 200, + height: 250, + min_height: 200, margin: { top: 10 }, toolbar: { widgets: [ @@ -231,17 +200,17 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { // Only label points if they are significant for some trait in the catalog, AND in high LD // with the top hit of interest { - field: "{{namespace[catalog]}}trait", + field: "catalog:trait", operator: "!=", value: null }, { - field: "{{namespace[catalog]}}log_pvalue", + field: "catalog:log_pvalue", operator: ">", value: 7.301 }, { - field: "{{namespace[ld]}}state", + field: "ld:state", operator: ">", value: 0.4 }, @@ -259,20 +228,11 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { ] }, data_layers: [ - LocusZoom.Layouts.get("data_layer", "significance", { unnamespaced: true }), - LocusZoom.Layouts.get("data_layer", "recomb_rate", { unnamespaced: true }), + LocusZoom.Layouts.get("data_layer", "significance"), + LocusZoom.Layouts.get("data_layer", "recomb_rate"), function() { var l = LocusZoom.Layouts.get("data_layer", "association_pvalues_catalog", { - unnamespaced: true, - fields: [ - "{{namespace[assoc]}}all", // special mock value for the custom source - "{{namespace[assoc]}}id", - "{{namespace[assoc]}}position", - "{{namespace[assoc]}}pvalue|neglog10_or_323", - "{{namespace[ld]}}state", "{{namespace[ld]}}isrefvar", - "{{namespace[catalog]}}rsid", "{{namespace[catalog]}}trait", "{{namespace[catalog]}}log_pvalue" - ], - id_field: "{{namespace[assoc]}}id", + id_field: "assoc:id", tooltip: { closable: true, show: { @@ -281,17 +241,17 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { hide: { "and": ["unhighlighted", "unselected"] }, - html: "{{{{namespace[assoc]}}id}}

" + + html: "{{assoc:id}}

" + window.model.tooltip_lztemplate.replace(/{{/g, "{{assoc:").replace(/{{assoc:#if /g, "{{#if assoc:").replace(/{{assoc:\/if}}/g, "{{/if}}") + "
" + - "Go to PheWAS" + - "{{#if {{namespace[catalog]}}rsid}}
See hits in GWAS catalog{{/if}}" + - "
{{#if {{namespace[ld]}}isrefvar}}LD Reference Variant{{#else}}Make LD Reference{{/if}}
" + "Go to PheWAS" + + "{{#if catalog:rsid}}
See hits in GWAS catalog{{/if}}" + + "
{{#if ld:isrefvar}}LD Reference Variant{{#else}}Make LD Reference{{/if}}
" }, - x_axis: { field: "{{namespace[assoc]}}position" }, + x_axis: { field: "assoc:position" }, y_axis: { axis: 1, - field: "{{namespace[assoc]}}pvalue|neglog10_or_323", + field: "assoc:pvalue|neglog10_or_323", floor: 0, upper_buffer: 0.1, min_extent: [0, 10] @@ -299,7 +259,7 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { }); l.behaviors.onctrlclick = [{ action: "link", - href: window.model.urlprefix+"/variant/{{{{namespace[assoc]}}chr}}-{{{{namespace[assoc]}}position}}-{{{{namespace[assoc]}}ref}}-{{{{namespace[assoc]}}alt}}" + href: window.model.urlprefix+"/variant/{{assoc:chr}}-{{assoc:position}}-{{assoc:ref}}-{{assoc:alt}}" }]; return l; }() @@ -309,7 +269,6 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { return base; }(), LocusZoom.Layouts.get("panel", "genes", { - unnamespaced: true, // proportional_height: 0.5, toolbar: { widgets: [{ @@ -319,18 +278,24 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { }, LocusZoom.Layouts.get('toolbar_widgets', 'gene_selector_menu')] }, data_layers: [ - LocusZoom.Layouts.get("data_layer", "genes_filtered", { - unnamespaced: true, - fields: ["{{namespace[gene]}}all"], - tooltip: { - html: ("

{{gene_name}}

" + - "
Gene ID: {{gene_id}}
" + - "
Transcript ID: {{transcript_id}}
" + - "
" + - "
More data on gnomAD/ExAC and Bravo
") - }, - - }) + function () { + const base = LocusZoom.Layouts.get("data_layer", "genes_filtered", { + tooltip: { + html: ("

{{gene_name}}

" + + "
Gene ID: {{gene_id}}
" + + "
Transcript ID: {{transcript_id}}
" + + "
" + + "
More data on gnomAD/ExAC and Bravo
") + }, + data_operations: [ + // Unlike the parent data layer, PheWeb does not fetch or display gene constraint from gnomAD + { type: 'fetch', from: ['gene'] }, + ] + }); + // Base LocusZoom genes layer specifies two namespaces. We can't *remove* a field through "Layouts.get" (because that just merges object keys). Instead we override the namespace section completely. + base.namespace = { gene: 'gene' }; + return base; + }(), ], }) ] diff --git a/pheweb/serve/static/variant.js b/pheweb/serve/static/variant.js index bb4b6c221..b979efe32 100644 --- a/pheweb/serve/static/variant.js +++ b/pheweb/serve/static/variant.js @@ -1,5 +1,10 @@ 'use strict'; +/** + * Code related to the single-variant page (including the PheWAS plot) + */ + + function deepcopy(obj) { return JSON.parse(JSON.stringify(obj)); } @@ -14,7 +19,7 @@ function custom_LocusZoom_Layouts_get(layout_type, layout_name, customizations) } else { var key_parts = key.split("."); var obj = layout; - for (var i=0; i < key_parts.length-1; i++) { + for (var i = 0; i < key_parts.length-1; i++) { // TODO: check that `obj` contains `key_parts[i]` obj = obj[key_parts[i]]; } @@ -32,6 +37,7 @@ LocusZoom.TransformationFunctions.add("percent", function(x) { return x + '%'; }); +// Override the LZ builtin scale function with a special version that handles more allowed fields LocusZoom.ScaleFunctions.add("effect_direction", function(parameters, input){ if (typeof input === "undefined"){ return null; @@ -48,7 +54,7 @@ LocusZoom.ScaleFunctions.add("effect_direction", function(parameters, input){ else if (input.or < 0) { return parameters['-'] || null; } } return null; -}); +}, true); (function() { // sort phenotypes @@ -107,28 +113,16 @@ LocusZoom.ScaleFunctions.add("effect_direction", function(parameters, input){ // Define data sources object // TODO: Can this be replaced with StaticSource + deepcopy? LocusZoom.Adapters.extend('PheWASLZ', 'PheWebSource', { - getData: function(state, fields, outnames, trans) { - // Override all parsing, namespacing, and field extraction mechanisms, and load data embedded within the page - trans = trans || []; - - var data = deepcopy(window.variant.phenos); //otherwise LZ adds attributes I don't want to the original data. - data.forEach(function(d, i) { - data[i].x = i; - data[i].id = i.toString(); - trans.forEach(function(transformation, t){ - if (typeof transformation === "function"){ - data[i][outnames[t]] = transformation(data[i][fields[t]]); - } - }); - }); - return function(chain) { - return {header: chain.header || {}, body: data}; - }.bind(this); + getData: function() { + const data = deepcopy(window.variant.phenos); + // Add a synthetic x field for plotting. (normal LZ does this automatically, but PheWeb bypasses that mechanism) + data.forEach((item, index) => item.x = index); + return data; } }); var data_sources = new LocusZoom.DataSources() - .add("phewas", ["PheWebSource", {url: '/this/is/not/used'}]); + .add("phewas", ["PheWebSource", {}]); var neglog10_significance_threshold = -Math.log10(0.05 / window.variant.phenos.length); @@ -136,25 +130,22 @@ LocusZoom.ScaleFunctions.add("effect_direction", function(parameters, input){ state: { variant: ['chrom', 'pos', 'ref', 'alt'].map(function(d) { return window.variant[d];}).join("-"), }, - dashboard: { - components: [ + toolbar: { + widgets: [ {type: "download", position: "right"}, {type: "download_png", position: "right"}, ], }, - min_height: 400, responsive_resize: true, mouse_guide: false, panels: [custom_LocusZoom_Layouts_get('panel', 'phewas', { - min_width: 640, // feels reasonable to me - margin: { top: 20, right: 40, bottom: 120, left: 50 }, + height: 375, + margin: { top: 20, right: 55, bottom: 130, left: 70 }, data_layers: [ LocusZoom.Layouts.get('data_layer', 'significance', { - unnamespaced: true, offset: neglog10_significance_threshold, }), custom_LocusZoom_Layouts_get('data_layer', 'phewas_pvalues', { - unnamespaced: true, id_field: 'idx', type: 'scatter', color: { @@ -175,6 +166,7 @@ LocusZoom.ScaleFunctions.add("effect_direction", function(parameters, input){ }, 'circle' ], + "x_axis.field": 'x', "y_axis.field": 'pval|neglog10_handle0', // handles pval=0 a little better "y_axis.upper_buffer": 0.1, "y_axis.min_extent": [0, neglog10_significance_threshold*1.05], // always show sig line @@ -206,7 +198,7 @@ LocusZoom.ScaleFunctions.add("effect_direction", function(parameters, input){ // Use categories as x ticks. "axes.x.ticks": window.first_of_each_category.map(function(pheno) { return { - style: {fill: pheno.color, "font-size":"11px", "font-weight":"bold", "text-anchor":"start"}, + style: {fill: pheno.color, "font-size": "12x", "font-weight": "bold", "text-anchor": "start"}, transform: "translate(15, 0) rotate(50)", text: pheno.category, x: pheno.idx