diff --git a/tx/README.md b/tx/README.md index c0f249e..35ad6e2 100644 --- a/tx/README.md +++ b/tx/README.md @@ -26,6 +26,8 @@ Add the `tx` section to your `config.json`: "enabled": true, "librarySource": "/path/to/library.yml", "cacheTimeout": 30, + "internalLimit" : 10000, + "externalLimit" : 1000, "expansionCacheSize": 1000, "expansionCacheMemoryThreshold": 0, "endpoints": [ @@ -57,14 +59,16 @@ Add the `tx` section to your `config.json`: ### Configuration Options -| Option | Type | Required | Description | -|--------|------|----------|-------------| -| `enabled` | boolean | Yes | Whether the module is enabled | -| `cacheTimeout` | integer | No | How many minutes to keep client side caches (for cache-id parameter). Default: 30 | -| `expansionCacheSize` | integer | No | Maximum number of expanded ValueSets to cache. Default: 1000 | -| `expansionCacheMemoryThreshold` | integer | No | Heap memory usage in MB that triggers evicting oldest half of expansion cache. 0 = disabled. Default: 0 | -| `librarySource` | string | Yes | Path to the YAML file that defines the terminology sources to load | -| `endpoints` | array | Yes | List of endpoint configurations (at least one required) | +| Option | Type | Required | Description | +|---------------------------------|---------|----------|---------------------------------------------------------------------------------------------------------| +| `enabled` | boolean | Yes | Whether the module is enabled | +| `cacheTimeout` | integer | No | How many minutes to keep client side caches (for cache-id parameter). Default: 30 | +| `expansionCacheSize` | integer | No | Maximum number of expanded ValueSets to cache. Default: 1000 | +| `expansionCacheMemoryThreshold` | integer | No | Heap memory usage in MB that triggers evicting oldest half of expansion cache. 0 = disabled. Default: 0 | +| `librarySource` | string | Yes | Path to the YAML file that defines the terminology sources to load | +| `internalLimit` | integer | No | Largest number of codes in internal expansions | +| `externalLimit` | integer | No | Largest number of codes the server will return in an expansion | +| `endpoints` | array | Yes | List of endpoint configurations (at least one required) | ### Endpoint Configuration diff --git a/tx/cs/cs-api.js b/tx/cs/cs-api.js index aa35d58..961f6f0 100644 --- a/tx/cs/cs-api.js +++ b/tx/cs/cs-api.js @@ -812,6 +812,32 @@ class CodeSystemFactoryProvider { return null; } + /** + * If the data available to the provider includes the definition of some supplements, + * then the provider has to declare them to the server by overriding this method. The + * method returns a list of CodeSystem resources, with jsonObj provided. The jsonObj + * in this case must include the correct metadata, with content = supplement, but need + * not include any actual content (which might be anticipated to be large). If the + * server sees a CodeSystem supplement with no content that comes from a provider + * then the server will use fillOutSupplement to ask for the details to be populated + * if a client has done something that means the server needs it (mostly, it doesn't) + * + * @returns {CodeSystem[]} + */ + async registerSupplements() { + return []; + } + + /** + * see comemnts for registerSupplements() + * + * @param {CodeSystem} supplement - the supplement to flesh out + * @returns void + */ + async fillOutSupplement(supplement) { + // nothing + } + /** * build and return a known concept map from the URL, if there is one. * diff --git a/tx/cs/cs-country.js b/tx/cs/cs-country.js index fde6bd8..61134ba 100644 --- a/tx/cs/cs-country.js +++ b/tx/cs/cs-country.js @@ -22,7 +22,6 @@ class CountryCodeServices extends CodeSystemProvider { super(opContext, supplements); this.codes = codes || []; this.codeMap = codeMap || new Map(); - this.supplements = []; } // Metadata methods diff --git a/tx/cs/cs-rxnorm.js b/tx/cs/cs-rxnorm.js index cb095c4..a9b3ec4 100644 --- a/tx/cs/cs-rxnorm.js +++ b/tx/cs/cs-rxnorm.js @@ -748,6 +748,9 @@ class RxNormTypeServicesFactory extends CodeSystemFactoryProvider { if (d.includes('_')) { d = d.substring(d.lastIndexOf('_') + 1); } + if (d.includes('-')) { + d = d.substring(0, d.lastIndexOf('-')); + } if (/^\d+$/.test(d)) { version = d; } diff --git a/tx/html/tx-template.html b/tx/html/tx-template.html index 3cede6e..3a82ecf 100644 --- a/tx/html/tx-template.html +++ b/tx/html/tx-template.html @@ -89,6 +89,7 @@ Capability Statement  |  Terminology Capabilities  |  Operations  |  + Problems  |    diff --git a/tx/library.js b/tx/library.js index c1cada9..5d8d8f0 100644 --- a/tx/library.js +++ b/tx/library.js @@ -758,6 +758,7 @@ class Library { cmp.close(); } } + } module.exports = { Library }; diff --git a/tx/problems.js b/tx/problems.js new file mode 100644 index 0000000..78baf62 --- /dev/null +++ b/tx/problems.js @@ -0,0 +1,68 @@ + +class ProblemFinder { + + constructor() { + this.map = new Map(); + } + + async scanValueSets(provider) { + let unknownVersions = {}; // system -> Set of versions not known to the server + for (let vsp of provider.valueSetProviders) { + let list = await vsp.listAllValueSets(); + for (let url of list) { + let vs = await vsp.fetchValueSet(url); + if (vs && vs.jsonObj.compose) { + await this.scanValueSet(vs.jsonObj.compose, unknownVersions, vs.jsonObj.status != 'retired'); + } + } + } + // Filter to only versions the server doesn't know about + for (const [system, vset] of Object.entries(unknownVersions)) { + for (let v of [...vset]) { + if (await provider.hasCsVersion(system, v)) { + vset.delete(v); + } + } + if (vset.size === 0) { + delete unknownVersions[system]; + } + } + return this.unknownVersionsHtml(unknownVersions); + } + + unknownVersionsHtml(unknownVersions) { + const entries = Object.entries(unknownVersions || {}); + if (entries.length === 0) { + return '

No unknown system versions found.

'; + } + entries.sort((a, b) => a[0].localeCompare(b[0])); + let html = ''; + for (const [system, vset] of entries) { + const versions = [...vset].sort((a, b) => a.localeCompare(b)).join('
'); + html += ``; + } + html += '
SystemUnknown Versions
${system}${versions}
'; + return html; + } + + async scanValueSet(compose, versions, active) { + for (let inc of compose.include || []) { + if (inc.system) { + if (active && inc.version) { + this.seeVersion(versions, inc.system, inc.version); + } + } + } + } + + seeVersion(versions, system, version) { + let set = versions[system]; + if (set == null) { + set = new Set(); + versions[system] = set; + } + set.add(version); + } +} + +module.exports = ProblemFinder; \ No newline at end of file diff --git a/tx/provider.js b/tx/provider.js index 866fa5b..28cb7ed 100644 --- a/tx/provider.js +++ b/tx/provider.js @@ -399,6 +399,20 @@ class Provider { return null; } + async hasCsVersion(system, version) { + for (let cs of this.codeSystems.values()) { + if (cs.url == system && cs.version == version) { + return true; + } + } + for (let cp of this.codeSystemFactories.values()) { + if (cp.system() == system && cp.version() == version) { + return true; + } + } + return false; + } + } module.exports = { Provider }; diff --git a/tx/tx.fhir.org.yml b/tx/tx.fhir.org.yml index af3a0f6..2be270e 100644 --- a/tx/tx.fhir.org.yml +++ b/tx/tx.fhir.org.yml @@ -26,6 +26,7 @@ sources: - snomed:sct_nl_20240930.cache - snomed:sct_uk_20230412.cache - snomed:sct_us_20230301.cache + - snomed:sct_us_20240301.cache - snomed:sct_us_20250901.cache - snomed:sct_test_20250814.cache - cpt:CodeSystem-cpt.db|cpt-2023-fragment-0.1.db diff --git a/tx/tx.js b/tx/tx.js index 8724ef3..a1dc65f 100644 --- a/tx/tx.js +++ b/tx/tx.js @@ -20,7 +20,7 @@ const packageJson = require("../package.json"); // Import workers const ReadWorker = require('./workers/read'); const SearchWorker = require('./workers/search'); -const { ExpandWorker } = require('./workers/expand'); +const { ExpandWorker, INTERNAL_DEFAULT_LIMIT, EXTERNAL_DEFAULT_LIMIT} = require('./workers/expand'); const { ValidateWorker } = require('./workers/validate'); const TranslateWorker = require('./workers/translate'); const LookupWorker = require('./workers/lookup'); @@ -49,6 +49,7 @@ const {convertResourceToR5} = require("./xversion/xv-resource"); const ClosureWorker = require("./workers/closure"); const {BundleXML} = require("./xml/bundle-xml"); const ConceptUsageTracker = require("./usage-tracker"); +const ProblemFinder = require("./problems"); // const {writeFileSync} = require("fs"); class TXModule { @@ -620,7 +621,7 @@ class TXModule { router.get('/ValueSet/\\$expand', async (req, res) => { const start = Date.now(); try { - let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n); + let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req)); await worker.handle(req, res, this.log); } finally { this.countRequest('$expand', Date.now() - start); @@ -629,7 +630,7 @@ class TXModule { router.post('/ValueSet/\\$expand', async (req, res) => { const start = Date.now(); try { - let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n); + let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req)); await worker.handle(req, res, this.log); } finally { this.countRequest('$expand', Date.now() - start); @@ -784,7 +785,7 @@ class TXModule { router.get('/ValueSet/:id/\\$expand', async (req, res) => { const start = Date.now(); try { - let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n); + let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req)); await worker.handleInstance(req, res, this.log); } finally { this.countRequest('$expand', Date.now() - start); @@ -793,7 +794,7 @@ class TXModule { router.post('/ValueSet/:id/\\$expand', async (req, res) => { const start = Date.now(); try { - let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n); + let worker = new ExpandWorker(req.txOpContext, this.log, req.txProvider, this.languages, this.i18n, this.internalLimit(req), this.externalLimit(req)); await worker.handleInstance(req, res, this.log); } finally { this.countRequest('$expand', Date.now() - start); @@ -893,6 +894,20 @@ class TXModule { } }); + router.get('/problems.html', async (req, res) => { + const start = Date.now(); + try { + let txhtml = new TxHtmlRenderer(new Renderer(req.txOpContext, req.txProvider), this.liquid); + const problemFinder = new ProblemFinder(); + const content = await problemFinder.scanValueSets(req.txProvider); + const html = await txhtml.renderPage('Problems', '

ValueSet dependencies on unknown CodeSystem/Versions

'+content, req.txEndpoint, req.txStartTime); + res.setHeader('Content-Type', 'text/html'); + res.send(html); + } finally { + this.countRequest('problems', Date.now() - start); + } + }); + // Metadata / CapabilityStatement router.get('/metadata', async (req, res) => { const start = Date.now(); @@ -1083,6 +1098,16 @@ class TXModule { } } + internalLimit(req) { + let isTest = req.header("User-Agent") == 'Tools/Java'; + if (this.config.internalLimit && !isTest) return this.config.internalLimit; else return INTERNAL_DEFAULT_LIMIT; + } + + externalLimit(req) { + let isTest = req.header("User-Agent") == 'Tools/Java'; + if (this.config.internalLimit && !isTest) return this.config.externalLimit; else return EXTERNAL_DEFAULT_LIMIT; + } + } module.exports = TXModule; \ No newline at end of file diff --git a/tx/usage-tracker.js b/tx/usage-tracker.js index 0768d4b..4fb0c7f 100644 --- a/tx/usage-tracker.js +++ b/tx/usage-tracker.js @@ -21,10 +21,13 @@ class ConceptUsageTracker { return c; } - async scanValueSet(compose) { + async scanValueSet(compose, versions, active) { let ok = false; for (let inc of compose.include || []) { if (inc.system) { + if (active && inc.version) { + this.seeVersion(versions, inc.system, inc.version); + } for (let c of inc.concept || []) { if (c.code) { ok = true; @@ -53,6 +56,15 @@ class ConceptUsageTracker { usages(system) { return this.map.get(system) || null; } + + seeVersion(versions, system, version) { + let set = versions[system]; + if (set == null) { + set = new Set(); + versions[system] = set; + } + set.add(version); + } } module.exports = ConceptUsageTracker; \ No newline at end of file diff --git a/tx/workers/expand.js b/tx/workers/expand.js index 3415c93..eaf5c01 100644 --- a/tx/workers/expand.js +++ b/tx/workers/expand.js @@ -20,9 +20,8 @@ const ValueSet = require("../library/valueset"); const {VersionUtilities} = require("../../library/version-utilities"); // Expansion limits (from Pascal constants) -const UPPER_LIMIT_NO_TEXT = 1000; -const UPPER_LIMIT_TEXT = 1000; -const INTERNAL_LIMIT = 10000; +const EXTERNAL_DEFAULT_LIMIT = 1000; +const INTERNAL_DEFAULT_LIMIT = 10000; const EXPANSION_DEAD_TIME_SECS = 30; const CACHE_WHEN_DEBUGGING = false; @@ -203,10 +202,14 @@ class ValueSetExpander { hasExclusions = false; requiredSupplements = new Set(); usedSupplements = new Set(); + internalLimit = INTERNAL_DEFAULT_LIMIT; + externalLimit = EXTERNAL_DEFAULT_LIMIT; constructor(worker, params) { this.worker = worker; this.params = params; + this.internalLimit = worker.internalLimit; + this.externalLimit = worker.externalLimit; this.csCounter = new Map(); } @@ -528,8 +531,10 @@ class ValueSetExpander { worker.additionalResources = this.worker.additionalResources; // we're going to let this one do more expansion for technical reasons let paramsInner = this.params.clone(); - paramsInner.limit = INTERNAL_LIMIT; + paramsInner.limit = this.internalLimit; let expander = new ValueSetExpander(worker, paramsInner); + expander.internalLimit = this.internalLimit; + expander.externalLimit = this.internalLimit; // it's deliberate that this is the internal limit let result = await expander.expand(vs, filter, false); if (result == null) { throw new Issue('error', 'not-found', null, 'VS_EXP_IMPORT_UNK', this.worker.i18n.translate('VS_EXP_IMPORT_UNK', this.params.httpLanguages, [uri]), 'unknown'); @@ -1157,13 +1162,9 @@ class ValueSetExpander { this.canBeHierarchy = !this.params.excludeNested; if (this.params.limit <= 0) { - if (!filter.isNull) { - this.limitCount = UPPER_LIMIT_TEXT; - } else { - this.limitCount = UPPER_LIMIT_NO_TEXT; - } + this.limitCount = this.externalLimit; } else { - this.limitCount = Math.min(this.params.limit, INTERNAL_LIMIT); + this.limitCount = Math.min(this.params.limit, this.externalLimit); } this.offset = this.params.offset; this.count = this.params.count; @@ -1622,6 +1623,9 @@ class ValueSetExpander { } class ExpandWorker extends TerminologyWorker { + internalLimit = INTERNAL_DEFAULT_LIMIT; + externalLimit = EXTERNAL_DEFAULT_LIMIT; + /** * @param {OperationContext} opContext - Operation context @@ -1630,8 +1634,11 @@ class ExpandWorker extends TerminologyWorker { * @param {LanguageDefinitions} languages - Language definitions * @param {I18nSupport} i18n - Internationalization support */ - constructor(opContext, log, provider, languages, i18n) { + constructor(opContext, log, provider, languages, + i18n, internalLimit = INTERNAL_DEFAULT_LIMIT, externalLimit = EXTERNAL_DEFAULT_LIMIT) { super(opContext, log, provider, languages, i18n); + this.externalLimit = externalLimit; + this.internalLimit = internalLimit; } /** @@ -1894,8 +1901,8 @@ class ExpandWorker extends TerminologyWorker { if (params.limit < -1) { params.limit = -1; - } else if (params.limit > UPPER_LIMIT_TEXT) { - params.limit = UPPER_LIMIT_TEXT; // can't ask for more than this externally, though you can internally + } else if (params.limit > EXTERNAL_DEFAULT_LIMIT) { + params.limit = EXTERNAL_DEFAULT_LIMIT; // can't ask for more than this externally, though you can internally } const filter = new SearchFilterText(params.filter); @@ -1943,9 +1950,8 @@ module.exports = { ImportedValueSet, ValueSetFilterContext, EmptyFilterContext, + EXTERNAL_DEFAULT_LIMIT, + INTERNAL_DEFAULT_LIMIT, TotalStatus, - UPPER_LIMIT_NO_TEXT, - UPPER_LIMIT_TEXT, - INTERNAL_LIMIT, EXPANSION_DEAD_TIME_SECS }; \ No newline at end of file diff --git a/tx/workers/validate.js b/tx/workers/validate.js index 7d1deea..b697627 100644 --- a/tx/workers/validate.js +++ b/tx/workers/validate.js @@ -379,7 +379,7 @@ class ValueSetChecker { if (!(ccf.property === 'concept' && ['is-a', 'descendent-of'].includes(ccf.op))) { if (!(await cs.doesFilter(ccf.property, ccf.op, ccf.value))) { throw new Issue('error', 'not-supported', "ValueSet.compose."+desc+".filter["+i+"]", 'FILTER_NOT_UNDERSTOOD', this.worker.i18n.translate('FILTER_NOT_UNDERSTOOD', - this.params.HTTPLanguages, [ccf.property, ccf.op, ccf.value, this.valueSet.url, cs.system]), "vs-invalid").handleAsOO(400); + this.params.HTTPLanguages, [ccf.property, ccf.op, ccf.value, this.valueSet.url, cs.system()]), "vs-invalid").handleAsOO(400); } } i++; diff --git a/tx/workers/worker.js b/tx/workers/worker.js index c1c749d..028072e 100644 --- a/tx/workers/worker.js +++ b/tx/workers/worker.js @@ -252,6 +252,8 @@ class TerminologyWorker { loadSupplements(url, version = '', statedSupplements) { const supplements = []; + // todo: look in provider for supplements + if (!this.additionalResources) { return supplements; } @@ -271,6 +273,9 @@ class TerminologyWorker { if (!(cs.isLangPack() || (statedSupplements && (statedSupplements.has(cs.url) || statedSupplements.has(cs.vurl))))) { continue; } + if (this.hasSupplement(cs, supplements)) { + continue; + } // Handle exact URL match (no version specified in supplements) if (supplementsUrl === url) { // If we're looking for a specific version, only include if no version in supplements URL @@ -900,6 +905,15 @@ class TerminologyWorker { console.log(error); } } + + hasSupplement(cs, supplements) { + for (let t of supplements) { + if (t.vurl == cs.vurl) { + return true; + } + } + return false; + } } module.exports = { diff --git a/tx/xversion/xv-parameters.js b/tx/xversion/xv-parameters.js index a723d6f..660bdc2 100644 --- a/tx/xversion/xv-parameters.js +++ b/tx/xversion/xv-parameters.js @@ -12,7 +12,14 @@ function parametersToR5(jsonObj, sourceVersion) { if (VersionUtilities.isR5Ver(sourceVersion)) { return jsonObj; // No conversion needed } - throw new Error(`Unsupported FHIR version: ${sourceVersion}`); + + const {convertResourceFromR5} = require("./xv-resource"); + for (let p of jsonObj.parameter) { + if (p.resource) { + p.resource = convertResourceFromR5(p.resource, sourceVersion); + } + } + return jsonObj; } /** diff --git a/tx/xversion/xv-resource.js b/tx/xversion/xv-resource.js index 4930646..409adba 100644 --- a/tx/xversion/xv-resource.js +++ b/tx/xversion/xv-resource.js @@ -1,11 +1,11 @@ -const {codeSystemFromR5} = require("./xv-codesystem"); -const {capabilityStatementFromR5} = require("./xv-capabiliityStatement"); -const {terminologyCapabilitiesFromR5} = require("./xv-terminologyCapabilities"); -const {valueSetFromR5} = require("./xv-valueset"); -const {conceptMapFromR5} = require("./xv-conceptmap"); -const {parametersFromR5} = require("./xv-parameters"); -const {operationOutcomeFromR5} = require("./xv-operationoutcome"); -const {bundleFromR5} = require("./xv-bundle"); +const {codeSystemFromR5, codeSystemToR5} = require("./xv-codesystem"); +const {capabilityStatementFromR5, capabilityStatementToR5} = require("./xv-capabiliityStatement"); +const {terminologyCapabilitiesFromR5, terminologyCapabilitiesToR5} = require("./xv-terminologyCapabilities"); +const {valueSetFromR5, valueSetToR5} = require("./xv-valueset"); +const {conceptMapFromR5, conceptMapToR5} = require("./xv-conceptmap"); +const {parametersFromR5, parametersToR5} = require("./xv-parameters"); +const {operationOutcomeFromR5, operationOutcomeToR5} = require("./xv-operationoutcome"); +const {bundleFromR5, bundleToR5} = require("./xv-bundle"); function convertResourceToR5(data, sourceVersion) { @@ -13,14 +13,14 @@ function convertResourceToR5(data, sourceVersion) { return data; } switch (data.resourceType) { - case "CodeSystem": return codeSystemFromR5(data, sourceVersion); - case "CapabilityStatement": return capabilityStatementFromR5(data, sourceVersion); - case "TerminologyCapabilities": return terminologyCapabilitiesFromR5(data, sourceVersion); - case "ValueSet": return valueSetFromR5(data, sourceVersion); - case "ConceptMap": return conceptMapFromR5(data, sourceVersion); - case "Parameters": return parametersFromR5(data, sourceVersion); - case "OperationOutcome": return operationOutcomeFromR5(data, sourceVersion); - case "Bundle": return bundleFromR5(data, sourceVersion); + case "CodeSystem": return codeSystemToR5(data, sourceVersion); + case "CapabilityStatement": return capabilityStatementToR5(data, sourceVersion); + case "TerminologyCapabilities": return terminologyCapabilitiesToR5(data, sourceVersion); + case "ValueSet": return valueSetToR5(data, sourceVersion); + case "ConceptMap": return conceptMapToR5(data, sourceVersion); + case "Parameters": return parametersToR5(data, sourceVersion); + case "OperationOutcome": return operationOutcomeToR5(data, sourceVersion); + case "Bundle": return bundleToR5(data, sourceVersion); default: return data; } }