diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pom.xml b/pom.xml index 947c4c01..274b6682 100644 --- a/pom.xml +++ b/pom.xml @@ -1,34 +1,41 @@ - + 4.0.0 org.reactome.release release-common-lib - 1.2.2 + 2.0.0 jar - Release Common Library - Common components used throughout the Release process. - https://reactome.org + ${project.groupId}:${project.artifactId} + Common components used throughout the Release process + https://github.com/reactome/release-common-lib + + Reactome + https://reactome.org + - GNU AFFERO GENERAL PUBLIC LICENSE - https://www.gnu.org/licenses/agpl-3.0.txt + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt Solomon Shorser - solomon.shorser@oicr.on.ca - Reactome - https://reactome.org + Ontario Institute for Cancer Research + https://oicr.on.ca + + Alumni + + Joel Weiser joel.weiser@oicr.on.ca - Reactome - https://reactome.org + Ontario Institute for Cancer Research + https://oicr.on.ca @@ -51,46 +58,65 @@ + + 2.5 + 3.8.0 + 2.2 + 4.5.13 + 2.17.0 + 1.1.1 + 5.8.2 + 1.8.2 + 3.12.4 + 1.0.1 + + + 11 UTF-8 - 2.1 - 5.1.0 - 1.1.0 - 3.6.0 + + 3.8.1 + 3.0.1 + 3.11.2 + 2.5.3 + 3.2.1 + 2.22.2 + 1.3.0 + 1.6.8 org.reactome.base - reactome-base - 1.2.23-SNAPSHOT + reactome-base-core + ${reactome.base.core.version} + org.apache.logging.log4j log4j-core - 2.11.0 - - - org.apache.httpcomponents - httpclient - 4.5.13 + ${log4j.version} + - javax.json - javax.json-api - 1.1.4 + com.googlecode.json-simple + json-simple + ${json.simple.version} + - org.glassfish - javax.json - 1.1.4 + commons-lang + commons-lang + ${commons.lang.version} + commons-net commons-net - 3.7.2 + ${commons.net.version} + org.hamcrest @@ -98,42 +124,42 @@ ${hamcrest.version} test + org.junit.jupiter junit-jupiter-api ${junit.version} test + org.junit.jupiter junit-jupiter-engine ${junit.version} test - - org.junit.vintage - junit-vintage-engine - ${junit.version} - test - + org.junit.platform junit-platform-launcher ${junit.platform.version} test + org.junit.platform junit-platform-runner ${junit.platform.version} test + org.mockito mockito-inline ${mockito.version} test + org.mockito mockito-junit-jupiter @@ -162,10 +188,10 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + ${maven.compiler.version} - 1.8 - 1.8 + ${jdk.version} + ${jdk.version} @@ -173,7 +199,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.0.0 + ${maven.javadoc.version} attach-javadocs @@ -182,13 +208,21 @@ + + 11 + -Xdoclint:none + + + https://docs.oracle.com/en/java/javase/11/docs/api/ + + org.apache.maven.plugins maven-source-plugin - 3.0.1 + ${maven.source.version} attach-sources @@ -203,7 +237,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + ${maven.gpg.version} sign-artifacts @@ -211,6 +245,10 @@ sign + + ${gpg.key} + ${gpg.key} + @@ -219,7 +257,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.8 + ${nexus-staging-maven-plugin} true ossrh @@ -232,17 +270,35 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + ${maven.surefire.version} + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.15 + + + integration-test + + integration-test + + + + verify + + verify + + + - + org.apache.maven.plugins maven-release-plugin - 2.5.3 + ${maven.release.version} true false @@ -252,14 +308,10 @@ - org.codehaus.mojo tidy-maven-plugin - 1.1.0 + ${maven.tidy.version} validate diff --git a/src/main/java/org/reactome/release/common/CustomLoggable.java b/src/main/java/org/reactome/release/common/CustomLoggable.java index 4244e29e..3e0c168b 100644 --- a/src/main/java/org/reactome/release/common/CustomLoggable.java +++ b/src/main/java/org/reactome/release/common/CustomLoggable.java @@ -31,7 +31,7 @@ public interface CustomLoggable * @param newAppenderName - the name of the new appender. * @param append - append, or not (truncate). * @param level - logging level. - * @return + * @return Logger with configurations from parameters set */ default Logger createLogger(String logFileName, String oldAppenderName, String newAppenderName, boolean append, Level level) { @@ -105,7 +105,7 @@ default Logger createLogger(String logFileName, String oldAppenderName, String n * @param level - logging level. * @param oldLogger - the old Logger - used to log the process of creating the new logger. * @param loggerContainerClassTypeName - name of class that invokes this method (no easy way to get that within the function via reflection, so please set this correctly). - * @return + * @return Logger with configurations from parameters set */ default Logger createLogger(String logFileName, String oldAppenderName, String newAppenderName, boolean append, Level level, Logger oldLogger, String loggerContainerClassTypeName) { diff --git a/src/main/java/org/reactome/release/common/ReleaseStep.java b/src/main/java/org/reactome/release/common/ReleaseStep.java index 61bb223f..ecf883fa 100644 --- a/src/main/java/org/reactome/release/common/ReleaseStep.java +++ b/src/main/java/org/reactome/release/common/ReleaseStep.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.gk.persistence.MySQLAdaptor; +import org.reactome.util.general.DBUtils; /** * This class provides the framework for a release step in Reactome's data-release-pipeline @@ -24,6 +25,8 @@ public abstract class ReleaseStep */ protected boolean testMode; + + /** * Parses a Properties object to obtain database connection information and return a database adaptor as a * MySQLAdaptor object. @@ -35,16 +38,8 @@ public abstract class ReleaseStep * @return Database adaptor as a MySQLAdaptor object * @throws SQLException Thrown if unable to create a MySQLAdaptor object to return */ - protected static MySQLAdaptor getMySQLAdaptorFromProperties(Properties props) throws SQLException - { - - String dbHost = props.getProperty("db.host", "localhost"); - String dbUser = props.getProperty("db.user"); - String dbPassword = props.getProperty("db.password"); - String dbName = props.getProperty("db.name"); - int dbPort = Integer.parseInt(props.getProperty("db.port", "3306")); - - return new MySQLAdaptor(dbHost, dbName, dbUser, dbPassword, dbPort); + protected static MySQLAdaptor getMySQLAdaptorFromProperties(Properties props) throws SQLException { + return DBUtils.getDbAdaptor("db", props); } /** diff --git a/src/main/java/org/reactome/release/common/dataretrieval/AuthenticatableFileRetriever.java b/src/main/java/org/reactome/release/common/dataretrieval/AuthenticatableFileRetriever.java index d7f1b0ed..11c380be 100644 --- a/src/main/java/org/reactome/release/common/dataretrieval/AuthenticatableFileRetriever.java +++ b/src/main/java/org/reactome/release/common/dataretrieval/AuthenticatableFileRetriever.java @@ -23,7 +23,7 @@ public AuthenticatableFileRetriever() /** * Set the user name. - * @param userName + * @param userName User name to set */ public void setUserName(String userName) { @@ -32,7 +32,7 @@ public void setUserName(String userName) /** * Set the password. - * @param password + * @param password Password to set */ public void setPassword(String password) { diff --git a/src/main/java/org/reactome/release/common/dataretrieval/FileRetriever.java b/src/main/java/org/reactome/release/common/dataretrieval/FileRetriever.java index 6572d56b..6a763ad2 100644 --- a/src/main/java/org/reactome/release/common/dataretrieval/FileRetriever.java +++ b/src/main/java/org/reactome/release/common/dataretrieval/FileRetriever.java @@ -1,33 +1,18 @@ package org.reactome.release.common.dataretrieval; -import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.SocketException; -import java.net.URI; -import java.net.URISyntaxException; +import java.io.*; +import java.net.*; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; -import org.apache.http.HttpStatus; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.HttpHostConnectException; -import org.apache.http.conn.UnsupportedSchemeException; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; @@ -138,18 +123,18 @@ else if (this.uri.getScheme().equals("ftp") || this.uri.getScheme().equals("sftp } else { - throw new UnsupportedSchemeException("URI "+this.uri.toString()+" uses an unsupported scheme: "+this.uri.getScheme()); + throw new Exception("URI "+this.uri.toString()+" uses an unsupported scheme: "+this.uri.getScheme()); } } catch (URISyntaxException e) { logger.error("Error creating download destination: " + this.destination, e); - e.printStackTrace(); + throw e; } catch (IOException e) { logger.error("Unable to create parent directory of download destination: " + path.toString(), e); - e.printStackTrace(); + throw e; } catch (Exception e) { @@ -160,12 +145,12 @@ else if (this.uri.getScheme().equals("ftp") || this.uri.getScheme().equals("sftp } - protected void doFtpDownload() throws SocketException, IOException, FileNotFoundException, Exception + protected void doFtpDownload() throws Exception { doFtpDownload(null, null); } - protected void doFtpDownload(String user, String password) throws SocketException, IOException, FileNotFoundException, Exception + protected void doFtpDownload(String user, String password) throws Exception { // user is "anonymous" if provided username is null/empty if (user == null || user.trim().equals("")) @@ -240,48 +225,40 @@ protected void writeInputStreamToFile(InputStream inStream) throws IOException, } } - - protected void doHttpDownload(Path path) throws HttpHostConnectException, IOException, Exception + protected void doHttpDownload(Path path) throws Exception, IOException { - this.doHttpDownload(path, HttpClientContext.create()); - } - - protected void doHttpDownload(Path path, HttpClientContext context) throws Exception, HttpHostConnectException, IOException - { - HttpGet get = new HttpGet(this.uri); - //Need to multiply by 1000 because timeouts are in milliseconds. - RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT) - .setConnectTimeout(1000 * (int)this.timeout.getSeconds()) - .setSocketTimeout(1000 * (int)this.timeout.getSeconds()) - .setConnectionRequestTimeout(1000 * (int)this.timeout.getSeconds()).build(); - - get.setConfig(config); - int retries = this.numRetries; boolean done = retries + 1 <= 0; while(!done) { - try( CloseableHttpClient client = HttpClients.createDefault(); - CloseableHttpResponse response = client.execute(get, context); - OutputStream outputFile = new FileOutputStream(path.toFile())) + try { - int statusCode = response.getStatusLine().getStatusCode(); + HttpURLConnection urlConnection = getHttpURLConnection(); + + //Need to multiply by 1000 because timeouts are in milliseconds. + int delayInMilliseconds = 1000 * (int)this.timeout.getSeconds(); + urlConnection.setConnectTimeout(delayInMilliseconds); + + urlConnection.setReadTimeout(delayInMilliseconds); + + int statusCode = urlConnection.getResponseCode(); // If status code was not 200, we should print something so that the users know that an unexpected response was received. - if (statusCode != HttpStatus.SC_OK) + if (statusCode != HttpURLConnection.HTTP_OK) { if (String.valueOf(statusCode).startsWith("4") || String.valueOf(statusCode).startsWith("5")) { - logger.error("Response code was 4xx/5xx: {}, Status line is: {}", statusCode, response.getStatusLine()); + logger.error("Response code was 4xx/5xx: {}, Status line is: {}", statusCode, urlConnection.getResponseMessage()); } else { - logger.warn("Response was not \"200\". It was: {}", response.getStatusLine()); + logger.warn("Response was not \"200\". It was: {}", urlConnection.getResponseMessage()); } } - response.getEntity().writeTo(outputFile); + + Files.copy(urlConnection.getInputStream(), path, StandardCopyOption.REPLACE_EXISTING); done = true; } - catch (ConnectTimeoutException e) + catch (SocketTimeoutException e) { // we will only be retrying the connection timeouts, defined as the time required to establish a connection. // we will not handle socket timeouts (inactivity that occurs after the connection has been established). @@ -295,12 +272,6 @@ protected void doHttpDownload(Path path, HttpClientContext context) throws Excep throw new Exception("Connection timed out. Number of retries ("+this.numRetries+") exceeded. No further attempts will be made.", e); } } - catch (HttpHostConnectException e) - { - logger.error("Could not connect to host {} !",get.getURI().getHost()); - e.printStackTrace(); - throw e; - } catch (IOException e) { logger.error("Exception caught: {}",e.getMessage()); throw e; @@ -308,6 +279,10 @@ protected void doHttpDownload(Path path, HttpClientContext context) throws Excep } } + protected HttpURLConnection getHttpURLConnection() throws IOException { + return (HttpURLConnection) this.uri.toURL().openConnection(); + } + public Duration getMaxAge() { return this.maxAge; @@ -369,5 +344,4 @@ public int getNumRetries() return this.numRetries; } -} - +} \ No newline at end of file diff --git a/src/main/java/org/reactome/release/common/dataretrieval/cosmic/COSMICFileRetriever.java b/src/main/java/org/reactome/release/common/dataretrieval/cosmic/COSMICFileRetriever.java index d5327b47..9c307534 100644 --- a/src/main/java/org/reactome/release/common/dataretrieval/cosmic/COSMICFileRetriever.java +++ b/src/main/java/org/reactome/release/common/dataretrieval/cosmic/COSMICFileRetriever.java @@ -1,25 +1,14 @@ package org.reactome.release.common.dataretrieval.cosmic; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; import org.reactome.release.common.dataretrieval.AuthenticatableFileRetriever; -import java.io.IOException; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; +import java.io.*; +import java.net.*; import java.util.Base64; - -import javax.json.Json; -import javax.json.JsonObject; -import javax.json.JsonReader; - -import org.apache.http.HttpStatus; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; +import java.util.stream.Collectors; /** * Get COSMIC data file. @@ -45,8 +34,7 @@ public COSMICFileRetriever() super(); } - private boolean retrieveAndSetCOSMICDownloadURL() throws UnsupportedEncodingException - { + private boolean retrieveAndSetCOSMICDownloadURL() throws IOException { boolean gotDownloadURLOK = false; // The NEW process to download from COSMIC requires a few steps. // 1) Generate a base64-encoded string of the username and password. @@ -55,41 +43,38 @@ private boolean retrieveAndSetCOSMICDownloadURL() throws UnsupportedEncodingExce // Encoded string. String encodedUsernamePassword = Base64.getEncoder().encodeToString((this.userName + ":" + this.password).getBytes("UTF-8")); - - HttpGet get = new HttpGet(this.uri); + + HttpURLConnection urlConnection = (HttpURLConnection) this.uri.toURL().openConnection(); //Need to multiply by 1000 because timeouts are in milliseconds. int delayInMilliseconds = 1000 * (int)this.timeout.getSeconds(); - RequestConfig config = RequestConfig.copy(RequestConfig.DEFAULT) - .setConnectTimeout(delayInMilliseconds) - .setSocketTimeout(delayInMilliseconds) - .setConnectionRequestTimeout(delayInMilliseconds).build(); - get.setConfig(config); - get.setHeader("Authorization", "Basic "+encodedUsernamePassword); + urlConnection.setConnectTimeout(delayInMilliseconds); + urlConnection.setReadTimeout(delayInMilliseconds); + urlConnection.setRequestProperty("Authorization", "Basic "+encodedUsernamePassword); + String downloadURL = null; - try( CloseableHttpClient client = HttpClients.createDefault(); - CloseableHttpResponse response = client.execute(get) ) - { - int statusCode = response.getStatusLine().getStatusCode(); - String responseString = EntityUtils.toString(response.getEntity()); + try { + int statusCode = urlConnection.getResponseCode(); + String content = getContent(urlConnection); // If status code was not 200, we should print something so that the users know that an unexpected response was received. switch (statusCode) { - case HttpStatus.SC_OK: - // Now we need to turn parse the JSON in responseString and extract the URL to download from. - JsonReader reader = Json.createReader(new StringReader(responseString)); - JsonObject responseObject = reader.readObject(); - downloadURL = responseObject.get("url").toString().replaceAll("\"", ""); - // Update this object's downloadURL to be the one that came back from the request - this.setDataURL(new URI(downloadURL)); - logger.info("COSMIC download URL has been set."); - gotDownloadURLOK = true; - // Call downloadData of FileRetriever to perform a "normal" download, now that the special URL has been set. - break; + case HttpURLConnection.HTTP_OK: + // Now we need to turn parse the JSON in responseString and extract the URL to download from. + + JSONParser jsonParser = new JSONParser(); + JSONObject responseObject = (JSONObject) jsonParser.parse(content); + downloadURL = responseObject.get("url").toString().replaceAll("\"", ""); + // Update this object's downloadURL to be the one that came back from the request + this.setDataURL(new URI(downloadURL)); + logger.info("COSMIC download URL has been set."); + gotDownloadURLOK = true; + // Call downloadData of FileRetriever to perform a "normal" download, now that the special URL has been set. + break; default: - logger.error("Non-200 status code: {} Response String is: {}", statusCode, responseString); - gotDownloadURLOK = false; - break; + logger.error("Non-200 response: {}", urlConnection.getResponseMessage()); + gotDownloadURLOK = false; + break; } } catch (IOException e) @@ -101,6 +86,9 @@ private boolean retrieveAndSetCOSMICDownloadURL() throws UnsupportedEncodingExce { logger.error("The URL from COSMIC might be malformed. URL is: \"{}\", Error message is: {}", downloadURL, e.getMessage() ); e.printStackTrace(); + } catch (ParseException e) { + logger.error("Error parsing JSON content. URL is: {}, Error message is {}", downloadURL, e.getMessage()); + e.printStackTrace(); } return gotDownloadURLOK; } @@ -118,5 +106,9 @@ protected void downloadData() throws UnsupportedEncodingException, Exception logger.warn("The COSMIC Download URL was not updated successfully, so file download was not attempted."); } } - + + private String getContent(HttpURLConnection urlConnection) throws IOException { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(urlConnection.getInputStream())); + return bufferedReader.lines().collect(Collectors.joining(System.lineSeparator())); + } } diff --git a/src/main/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessor.java b/src/main/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessor.java index fab4ebd8..260bd3fc 100644 --- a/src/main/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessor.java +++ b/src/main/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessor.java @@ -1,19 +1,18 @@ package org.reactome.util.ensembl; +import java.io.BufferedReader; import java.io.IOException; -import java.net.URI; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URLConnection; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.ParseException; -import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,7 +34,7 @@ public final class EnsemblServiceResponseProcessor /** * Constructs a new EnsemblServiceResponseProcessor object with the specified logger - * to record information about HttpResponse objects from the EnsEMBL service + * to record information about responses from the EnsEMBL service * @param logger Logger object to record information about processed responses */ public EnsemblServiceResponseProcessor(Logger logger) @@ -50,7 +49,7 @@ public EnsemblServiceResponseProcessor(Logger logger) /** * Constructs a new EnsemblServiceResponseProcessor object with a default logger - * to record information about HttpResponse objects from the EnsEMBL service + * to record information about responses from the EnsEMBL service */ public EnsemblServiceResponseProcessor() { @@ -60,30 +59,15 @@ public EnsemblServiceResponseProcessor() /** * Processes the HttpResponse from the queried EnsEMBL service, logs relevant information for the end-user, and * returns the important information of the response as an EnsemblServiceResult object - * @param response HttpResponse from the EnsEMBL service - * @param originalURI The URI of the EnsEMBL service queried (e.g rest.ensembl.org or rest.ensemblgenomes.org) + * @param urlConnection URLConnection from the EnsEMBL service * @return EnsemblServiceResult object containing the relevant information from the response - * @deprecated use {@link #processResponse(HttpResponse) instead} + * @throws IOException Thrown if unable to get response code or message from urlConnection */ - @Deprecated - public EnsemblServiceResult processResponse(HttpResponse response, URI originalURI) - { - return processResponse(response); - } - - /** - * Processes the HttpResponse from the queried EnsEMBL service, logs relevant information for the end-user, and - * returns the important information of the response as an EnsemblServiceResult object - * @param response HttpResponse from the EnsEMBL service - * @return EnsemblServiceResult object containing the relevant information from the response - */ - public EnsemblServiceResult processResponse(HttpResponse response) - { - EnsemblServiceResult result = response.containsHeader("Retry-After") ? - processResponseWithRetryAfter(response) : - processResponseWhenNotOverQueryQuota(response); - - processXRateLimitRemaining(response); + public EnsemblServiceResult processResponse(HttpURLConnection urlConnection) throws IOException { + EnsemblServiceResult result = urlConnection.getHeaderFields().get("Retry-After") != null ? + processResponseWithRetryAfter(urlConnection) : + processResponseWhenNotOverQueryQuota(urlConnection); + processXRateLimitRemaining(urlConnection); return result; } @@ -136,7 +120,7 @@ private void initializeTimeoutRetriesRemaining() } /** - * Process the query's response object when the response contains the header "Retry-After" which is most likely + * Process the query's URLConnection object when the response contains the header "Retry-After" which is most likely * to happen if too many requests are sent in a fixed time, using up our quota with the service resulting in a * need to wait before more requests can be sent. * @@ -145,20 +129,18 @@ private void initializeTimeoutRetriesRemaining() * before retrying. * * The response's status message, reason phrase, and headers will also be logged for debugging. - * @param response Response object to process + * @param urlConnection urlConnection URLConnection from the EnsEMBL service * @return EnsemblServiceResult object with the response status, isOkToRetry, and wait time to retry set */ - EnsemblServiceResult processResponseWithRetryAfter(HttpResponse response) - { - logger.debug("Response message: {} ; Reason code: {}; Headers: {}", - response.getStatusLine().toString(), - response.getStatusLine().getReasonPhrase(), - getHeaders(response) + EnsemblServiceResult processResponseWithRetryAfter(HttpURLConnection urlConnection) throws IOException { + logger.debug("Response message: {} ; Headers: {}", + urlConnection.getResponseMessage(), + getHeaders(urlConnection) ); EnsemblServiceResult result = this.new EnsemblServiceResult(); - result.setStatus(response.getStatusLine().getStatusCode()); - result.setWaitTime(processWaitTime(response)); + result.setStatus(urlConnection.getResponseCode()); + result.setWaitTime(processWaitTime(urlConnection)); result.setOkToRetry(timesWaitedThresholdNotMet()); if (result.isOkToRetry()) @@ -170,21 +152,21 @@ EnsemblServiceResult processResponseWithRetryAfter(HttpResponse response) } /** - * Processes the query's response object when the response does not contain the header "Retry-After". This method - * will attempt to retrieve the content, and return it as the result value in an EnsemblServiceResult object. + * Processes the query's URLConnection object when the response does not contain the header "Retry-After". This + * method will attempt to retrieve the content, and return it as the result value in an EnsemblServiceResult + * object. * * Certain HTTP response error codes (e.g. 400, 404, 500, 504) will cause the error to be logged and a * EnsemblServiceResult object without content set in the result value to be returned. - * @param response Response object to process + * @param urlConnection HttpURLConnection from the EnsEMBL service * @return EnsemblServiceResult object with the content set as its result value if the content was obtained */ - EnsemblServiceResult processResponseWhenNotOverQueryQuota(HttpResponse response) - { + EnsemblServiceResult processResponseWhenNotOverQueryQuota(HttpURLConnection urlConnection) throws IOException { EnsemblServiceResult result = this.new EnsemblServiceResult(); - result.setStatus(response.getStatusLine().getStatusCode()); - switch (response.getStatusLine().getStatusCode()) + result.setStatus(urlConnection.getResponseCode()); + switch (urlConnection.getResponseCode()) { - case HttpStatus.SC_GATEWAY_TIMEOUT: + case HttpURLConnection.HTTP_GATEWAY_TIMEOUT: logger.error("Request timed out! {} retries remaining", timeoutRetriesRemaining); timeoutRetriesRemaining--; @@ -198,28 +180,27 @@ EnsemblServiceResult processResponseWhenNotOverQueryQuota(HttpResponse response) initializeTimeoutRetriesRemaining(); } break; - case HttpStatus.SC_OK: - result.setResult(parseContent(response)); + case HttpURLConnection.HTTP_OK: + result.setResult(parseContent(urlConnection)); break; - case HttpStatus.SC_NOT_FOUND: + case HttpURLConnection.HTTP_NOT_FOUND: logger.error("Response code 404 ('Not found') received: {}", - response.getStatusLine().getReasonPhrase() + urlConnection.getResponseMessage() ); // If we got 404, don't retry. break; - case HttpStatus.SC_INTERNAL_SERVER_ERROR: - logger.error("Error 500 detected! Message: {}",response.getStatusLine().getReasonPhrase()); + case HttpURLConnection.HTTP_INTERNAL_ERROR: + logger.error("Error 500 detected! Message: {}", urlConnection.getResponseMessage()); // If we get 500 error then we should just get out of here. Maybe throw an exception? break; - case HttpStatus.SC_BAD_REQUEST: - logger.trace("Response code was 400 ('Bad request'). Message from server: {}", parseContent(response)); + case HttpURLConnection.HTTP_BAD_REQUEST: + logger.trace("Response code was 400 ('Bad request'). Message from server: {}", parseContent(urlConnection)); break; default: // Log any other kind of response. - result.setResult(parseContent(response)); - logger.info("Unexpected response {} with message: {}", - response.getStatusLine().getStatusCode(), - response.getStatusLine().getReasonPhrase() + result.setResult(parseContent(urlConnection)); + logger.info("Unexpected response: {}", + urlConnection.getResponseMessage() ); break; } @@ -232,17 +213,16 @@ EnsemblServiceResult processResponseWhenNotOverQueryQuota(HttpResponse response) * "X-RateLimit-Remaining". * * The number of requests remaining will be logged for debugging if it is a multiple of 1000. If no - * "X-RateLimit-Remaining" header is found in the response object, its absence will be logged along with + * "X-RateLimit-Remaining" header is found in the URLConnection, its absence will be logged along with * the HTTP response code received, the response headers received, and the last known number of requests * remaining. - * @param response Response object from which to obtain the number of requests remaining + * @param urlConnection URLConnection from the EnsEMBL service */ - void processXRateLimitRemaining(HttpResponse response) - { - if (response.containsHeader("X-RateLimit-Remaining")) + void processXRateLimitRemaining(HttpURLConnection urlConnection) throws IOException { + if (urlConnection.getHeaderFields().get("X-RateLimit-Remaining") != null) { EnsemblServiceResponseProcessor.numRequestsRemaining.set( - parseIntegerHeaderValue(response, "X-RateLimit-Remaining") + parseIntegerHeaderValue(urlConnection, "X-RateLimit-Remaining") ); if (EnsemblServiceResponseProcessor.numRequestsRemaining.get() % 1000 == 0) { @@ -256,8 +236,8 @@ void processXRateLimitRemaining(HttpResponse response) "Headers returned are: {} " + System.lineSeparator() + "Last known value for remaining was {}", - response.getStatusLine().toString(), - getHeaders(response), + urlConnection.getResponseMessage(), + getHeaders(urlConnection), EnsemblServiceResponseProcessor.numRequestsRemaining ); } @@ -267,13 +247,13 @@ void processXRateLimitRemaining(HttpResponse response) * Returns the duration to wait before retrying the query. The wait time is determined from the "Retry-After" * header in the query response and is multiplied by the number of times the query has been requested to wait. * This is to give the increasing server buffer time on each failure requesting the query waits. - * @param response Response object from which to obtain the base wait time + * @param urlConnection URLConnection from the EnsEMBL service * @return Wait time as Duration object ("Retry-After" value multiplied by the number of times the query has * requested to wait). */ - private Duration processWaitTime(HttpResponse response) + private Duration processWaitTime(HttpURLConnection urlConnection) { - Duration waitTime = Duration.ofSeconds(parseIntegerHeaderValue(response,"Retry-After")); + Duration waitTime = Duration.ofSeconds(parseIntegerHeaderValue(urlConnection,"Retry-After")); logger.warn("The server told us to wait, so we will wait for {} * {} before trying again.", waitTime, getWaitMultiplier() @@ -307,19 +287,21 @@ private boolean timesWaitedThresholdNotMet() } /** - * Parses and returns the content from the response object passed. Content is parsed as UTF-8. A stacktrace is - * printed to STDERR and an empty String returned if an exception occurs during parsing of the response content. - * @param response Response object to query for content - * @return Content as String from the response object passed. An empty String if an exception occurs during + * Parses and returns the content from the URLConnection object passed. Content is parsed as UTF-8. A stacktrace + * is printed to STDERR and an empty String returned if an exception occurs during parsing of the response content. + * @param urlConnection URLConnection from the EnsEMBL service + * @return Content as String from the URLConnection object passed. An empty String if an exception occurs during * parsing of the content. */ - private String parseContent(HttpResponse response) + private String parseContent(HttpURLConnection urlConnection) { try { - return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + return new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining(System.lineSeparator())); } - catch (ParseException | IOException e) + catch (IOException e) { e.printStackTrace(); return ""; @@ -328,23 +310,28 @@ private String parseContent(HttpResponse response) /** * Returns the value of a header, which is an integer, as an integer. - * @param response Response object to query for headers + * @param urlConnection URLConnection from the EnsEMBL service * @param header Specific header for which to get the value (which should be an integer) - * @return Integer value of the header passed for the response object passed + * @return Integer value of the header passed for the URLConnection object passed */ - private static int parseIntegerHeaderValue(HttpResponse response, String header) throws NumberFormatException + private static int parseIntegerHeaderValue(URLConnection urlConnection, String header) throws NumberFormatException { - return Integer.parseInt(response.getHeaders(header)[0].getValue()); + return Integer.parseInt(urlConnection.getHeaderFields().get(header).get(0)); } /** - * Returns the headers of the response object as a list of Strings - * @param response Response object from which to get the headers - * @return List of Strings representing the headers from the passed response object + * Returns the headers of the URLConnection object as a list of Strings + * @param urlConnection URLConnection from the EnsEMBL service + * @return List of Strings representing the headers from the passed URLConnection object */ - private List getHeaders(HttpResponse response) + private List getHeaders(URLConnection urlConnection) { - return Arrays.stream(response.getAllHeaders()).map(Object::toString).collect(Collectors.toList()); + List headerNamesAndValues = new ArrayList<>(); + for (String headerName : urlConnection.getHeaderFields().keySet()) { + String headerValues = urlConnection.getHeaderFields().get(headerName).stream().collect(Collectors.joining(", ")); + headerNamesAndValues.add(headerName + ": " + headerValues); + } + return headerNamesAndValues; } /** diff --git a/src/main/java/org/reactome/util/general/DBUtils.java b/src/main/java/org/reactome/util/general/DBUtils.java new file mode 100644 index 00000000..9ccf8143 --- /dev/null +++ b/src/main/java/org/reactome/util/general/DBUtils.java @@ -0,0 +1,31 @@ +package org.reactome.util.general; + +import org.gk.persistence.MySQLAdaptor; + +import java.sql.SQLException; +import java.util.Properties; + +/** + * @author Joel Weiser (joel.weiser@oicr.on.ca) + * Created 4/1/2024 + */ +public class DBUtils { + + public static MySQLAdaptor getCuratorDbAdaptor(Properties configProperties) throws SQLException { + return getDbAdaptor("curator.database", configProperties); + } + + public static MySQLAdaptor getReleaseDbAdaptor(Properties configProperties) throws SQLException { + return getDbAdaptor("release.database", configProperties); + } + + public static MySQLAdaptor getDbAdaptor(String prefix, Properties configProperties) throws SQLException { + return new MySQLAdaptor( + configProperties.getProperty(prefix + ".host", "localhost"), + configProperties.getProperty(prefix + ".name"), + configProperties.getProperty(prefix + ".user", "root"), + configProperties.getProperty(prefix + ".password", "root"), + Integer.parseInt(configProperties.getProperty(prefix + ".port", "3306")) + ); + } +} diff --git a/src/main/java/org/reactome/util/general/GUnzipCallable.java b/src/main/java/org/reactome/util/general/GUnzipCallable.java index d977d496..e50a8c40 100644 --- a/src/main/java/org/reactome/util/general/GUnzipCallable.java +++ b/src/main/java/org/reactome/util/general/GUnzipCallable.java @@ -16,7 +16,7 @@ * It made more sense to implement Runnable, but ExecutorService.invokeAll only accepts Callables. * The return of this calling call() on this object will be true if unzipping doesn't completely fail, * but you should not rely on this. The source file does *not* get removed by this process. - *

+ * * Example usage: 
 	GUnzipCallable datUnzipper1 = new GUnzipCallable(Paths.get("large_file_1.dat.gz"), Paths.get("./large_file_1.dat"));
@@ -50,7 +50,8 @@ public GUnzipCallable(Path src, Path targ)
 	
 	/**
 	 * Decompress a gzip file.
-	 * @throws IOException
+	 * @throws IOException Thrown if unable to create a file stream for the source (zipped file) or
+	 * target (unzipped file) or to read from the source or write to the target.
 	 */
 	public void decompressGzip() throws IOException
 	{
diff --git a/src/main/java/org/reactome/util/general/MandatoryProperties.java b/src/main/java/org/reactome/util/general/MandatoryProperties.java
index db340f52..3175d6b4 100644
--- a/src/main/java/org/reactome/util/general/MandatoryProperties.java
+++ b/src/main/java/org/reactome/util/general/MandatoryProperties.java
@@ -3,7 +3,8 @@
 import java.util.Properties;
 
 /**
- * A specialised implementation of Properties that will throw an exception if a requested property key is missing or has no value. 
+ * A specialised implementation of Properties that will throw an exception if a requested property key is missing or
+ * has no value.
  * @author sshorser
  *
  */
diff --git a/src/test/java/org/reactome/release/common/dataretrieval/COSMICFileRetrieverIT.java b/src/test/java/org/reactome/release/common/dataretrieval/COSMICFileRetrieverIT.java
index 9e603da4..b084d14e 100644
--- a/src/test/java/org/reactome/release/common/dataretrieval/COSMICFileRetrieverIT.java
+++ b/src/test/java/org/reactome/release/common/dataretrieval/COSMICFileRetrieverIT.java
@@ -17,8 +17,6 @@
 
 public class COSMICFileRetrieverIT
 {
-
-	
 	@Test
 	public void testGetCOSMICData() throws URISyntaxException, FileNotFoundException, IOException
 	{
@@ -31,7 +29,7 @@ public void testGetCOSMICData() throws URISyntaxException, FileNotFoundException
 		retriever.setPassword(props.getProperty("password"));
 		
 		// classification.csv is a pretty small file, ~2MB I think, so a good choice for testing downloads.
-		retriever.setDataURL(new URI("https://cancer.sanger.ac.uk/cosmic/file_download/GRCh38/cosmic/v92/classification.csv"));
+		retriever.setDataURL(new URI("https://cancer.sanger.ac.uk/cosmic/file_download/GRCh38/cosmic/v99/classification.csv"));
 		retriever.setMaxAge(Duration.ofSeconds(10));
 		retriever.setTimeout(Duration.ofSeconds(100));
 		retriever.setFetchDestination("/tmp/cosmic_classification.csv");
@@ -48,4 +46,4 @@ public void testGetCOSMICData() throws URISyntaxException, FileNotFoundException
 			fail();
 		}
 	}
-}
+}
\ No newline at end of file
diff --git a/src/test/java/org/reactome/release/common/dataretrieval/TestFileRetrieval.java b/src/test/java/org/reactome/release/common/dataretrieval/TestFileRetrieval.java
index 7cb621ed..e630b8c2 100644
--- a/src/test/java/org/reactome/release/common/dataretrieval/TestFileRetrieval.java
+++ b/src/test/java/org/reactome/release/common/dataretrieval/TestFileRetrieval.java
@@ -5,12 +5,12 @@
 
 import static org.junit.Assert.assertTrue;
 
-import static org.mockito.ArgumentMatchers.any;
+import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.anyString;
 
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.io.InputStream;
+import java.net.SocketTimeoutException;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -19,16 +19,7 @@
 import java.util.Random;
 
 import org.apache.commons.net.ftp.FTPClient;
-import org.apache.http.HttpEntity;
-import org.apache.http.StatusLine;
-import org.apache.http.client.ClientProtocolException;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.conn.ConnectTimeoutException;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.protocol.HttpContext;
+
 import org.junit.Before;
 import org.junit.Test;
 
@@ -50,18 +41,7 @@ public class TestFileRetrieval {
 	
 	@Mock
 	URI mockUri;
-	
-	@Mock
-	CloseableHttpClient mockClient;
-	
-	@Mock
-	CloseableHttpResponse mockResponse;
 
-	@Mock
-	StatusLine mockStatusLine;
-	
-	HttpEntity entity = new ByteArrayEntity(MESSAGE_CONTENT.getBytes());
-	
 	@Before
 	public void setup()
 	{
@@ -69,48 +49,38 @@ public void setup()
 	}
 	
 	/**
-	 * Test method for {@link org.reactome.addlinks.dataretrieval.FileRetriever#fetchData()}.
+	 * Test method for {@link org.reactome.release.common.dataretrieval.FileRetriever#fetchData()}.
 	 * @throws Exception 
 	 */
 	@Test
 	public void testFetchData() throws Exception
 	{
-		try(MockedStatic mockedStatic = Mockito.mockStatic(HttpClients.class);)
-		{
-			DataRetriever retriever = new FileRetriever();
-			//retrieve google - it should be pretty easy.
-			URI uri = new URI("http://www.google.com");
-			retriever.setDataURL(uri);
-			
-			Mockito.when(mockResponse.getEntity()).thenReturn(entity);
-			Mockito.when(mockResponse.getStatusLine()).thenReturn(mockStatusLine);
-			Mockito.when(mockClient.execute(any(HttpUriRequest.class))).thenReturn(mockResponse);
-			Mockito.when(mockClient.execute(any(HttpUriRequest.class), (HttpContext) any(HttpContext.class))).thenReturn(mockResponse);
-			
-			Mockito.when(HttpClients.createDefault()).thenReturn(mockClient);
-			
-			String dest = "/tmp/testFetchData_"+ String.valueOf((new Random()).nextInt(Integer.MAX_VALUE));
-			retriever.setFetchDestination(dest);
-			Duration age = Duration.of(5, ChronoUnit.SECONDS);
-			retriever.setMaxAge(age);
-			
-			retriever.fetchData();
-			assertTrue(Files.exists(Paths.get(dest)));
-			
-			//Sleep for 6 seconds, and then re-download because the file is stale (MAX AGE was 5 seconds).
-			Thread.sleep(Duration.of(6, ChronoUnit.SECONDS).toMillis());
-			retriever.fetchData();
-			assertTrue(Files.exists(Paths.get(dest)));
-			//now set a longer maxAge.
-			age = Duration.of(100, ChronoUnit.SECONDS);
-			retriever.setMaxAge(age);
-			// this time, the file will not be stale (because maxAge is larger) so nothing will be downloaded.
-			retriever.fetchData();
-			//check that the file exists.
-			assertTrue(Files.exists(Paths.get(dest)));
-		}
+		DataRetriever retriever = new FileRetriever();
+		//retrieve google - it should be pretty easy.
+		URI uri = new URI("https://www.google.com");
+		retriever.setDataURL(uri);
+
+		String dest = "/tmp/testFetchData_"+ String.valueOf((new Random()).nextInt(Integer.MAX_VALUE));
+		retriever.setFetchDestination(dest);
+		Duration age = Duration.of(5, ChronoUnit.SECONDS);
+		retriever.setMaxAge(age);
+
+		retriever.fetchData();
+		assertTrue(Files.exists(Paths.get(dest)));
+
+		//Sleep for 6 seconds, and then re-download because the file is stale (MAX AGE was 5 seconds).
+		Thread.sleep(Duration.of(6, ChronoUnit.SECONDS).toMillis());
+		retriever.fetchData();
+		assertTrue(Files.exists(Paths.get(dest)));
+		//now set a longer maxAge.
+		age = Duration.of(100, ChronoUnit.SECONDS);
+		retriever.setMaxAge(age);
+		// this time, the file will not be stale (because maxAge is larger) so nothing will be downloaded.
+		retriever.fetchData();
+		//check that the file exists.
+		assertTrue(Files.exists(Paths.get(dest)));
 	}
-	
+
 	/**
 	 * Test retrieving FTP data.
 	 * @throws Exception
@@ -153,7 +123,7 @@ public void prepare(FTPClient localMockFTPClient, Context context) throws Throwa
 		}
 	}
 	
-	/**
+/**
 	 * Test FTP Error handling.
 	 * @throws Exception
 	 */
@@ -202,75 +172,67 @@ public void prepare(FTPClient localMockFTPClient, Context context) throws Throwa
 	
 	/**
 	 * Test HTTP Error handling
-	 * @throws ClientProtocolException
-	 * @throws IOException
 	 * @throws Exception
 	 */
 	@Test
-	public void testHttpErr() throws ClientProtocolException, IOException, Exception
+	public void testHttpErr() throws Exception
 	{
-		try(MockedStatic mockedStatic = Mockito.mockStatic(HttpClients.class);)
+		FileRetriever retriever = Mockito.spy(FileRetriever.class);
+		//retrieve google - it should be pretty easy.
+		retriever.setDataURL(new URI("https://www.google.com"));
+		String dest = "/tmp/testFetchData_"+ String.valueOf((new Random()).nextInt(Integer.MAX_VALUE));
+		retriever.setFetchDestination(dest);
+		retriever.setMaxAge(Duration.of(1,ChronoUnit.SECONDS));
+		retriever.setNumRetries(0);
+		retriever.setTimeout(Duration.of(1, ChronoUnit.SECONDS));
+
+		try
 		{
-			Mockito.when(mockClient.execute(any(HttpUriRequest.class), any(HttpContext.class))).thenThrow(new ClientProtocolException("MOCK Generic Error"));
-			
-			Mockito.when(HttpClients.createDefault()).thenReturn(mockClient);
-			
-			DataRetriever retriever = new FileRetriever();
-			//retrieve google - it should be pretty easy.
-			URI uri = new URI("http://www.google.com");
-			retriever.setDataURL(uri);
-			String dest = "/tmp/testFetchData_"+ String.valueOf((new Random()).nextInt(Integer.MAX_VALUE));
-			retriever.setFetchDestination(dest);
-			Duration age = Duration.of(1,ChronoUnit.SECONDS);
-			retriever.setMaxAge(age);
-			((FileRetriever)retriever).setNumRetries(0);
-			((FileRetriever)retriever).setTimeout(Duration.of(1, ChronoUnit.SECONDS));
-			try
-			{
-				retriever.fetchData();
-			}
-			catch (Exception e)
-			{
-				e.printStackTrace();
-				assertTrue(e.getMessage().contains("MOCK Generic Error"));
-			}
+			Mockito.doThrow(new SocketTimeoutException("Dummy text"))
+				.when(retriever).getHttpURLConnection();
+			retriever.fetchData();
+
+			// Test fails here since an fetch data should throw an exception going to the catch block
+			// (that is, fetching data on previous line should throw an exception due to the mock)
+			fail();
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+			assertTrue(e.getMessage().contains("No further attempts will be made"));
 		}
 	}
 	
 	/**
 	 * Test HTTP Retry.
-	 * @throws ClientProtocolException
-	 * @throws IOException
 	 * @throws Exception
 	 */
 	@Test
-	public void testHttpRetry() throws ClientProtocolException, IOException, Exception
+	public void testHttpRetry() throws Exception
 	{
-		try(MockedStatic mockedStatic = Mockito.mockStatic(HttpClients.class);)
+		FileRetriever retriever = Mockito.spy(FileRetriever.class);
+		//retrieve google - it should be pretty easy.
+		retriever.setDataURL(new URI("https://www.google.com"));
+		String dest = "/tmp/testFetchData_"+ String.valueOf((new Random()).nextInt(Integer.MAX_VALUE));
+		retriever.setFetchDestination(dest);
+		retriever.setMaxAge(Duration.of(1,ChronoUnit.SECONDS));
+		retriever.setNumRetries(1);
+		retriever.setTimeout(Duration.of(1, ChronoUnit.SECONDS));
+
+		try
 		{
-			Mockito.when(mockClient.execute(any(HttpUriRequest.class), any(HttpContext.class))).thenThrow(new ConnectTimeoutException("MOCK Timeout Error"));
-			
-			Mockito.when(HttpClients.createDefault()).thenReturn(mockClient);
-			
-			DataRetriever retriever = new FileRetriever();
-			//retrieve google - it should be pretty easy.
-			URI uri = new URI("http://www.google.com");
-			retriever.setDataURL(uri);
-			String dest = "/tmp/testFetchData_"+ String.valueOf((new Random()).nextInt(Integer.MAX_VALUE));
-			retriever.setFetchDestination(dest);
-			Duration age = Duration.of(1,ChronoUnit.SECONDS);
-			retriever.setMaxAge(age);
-			((FileRetriever)retriever).setNumRetries(1);
-			((FileRetriever)retriever).setTimeout(Duration.of(1, ChronoUnit.SECONDS));
-			try
-			{
-				retriever.fetchData();
-			}
-			catch (Exception e)
-			{
-				e.printStackTrace();
-				assertTrue(e.getMessage().contains("Connection timed out. Number of retries (1) exceeded. No further attempts will be made."));
-			}
+			Mockito.doThrow(new SocketTimeoutException("Dummy text"))
+				.when(retriever).getHttpURLConnection();
+			retriever.fetchData();
+
+			// Test fails here since an fetch data should throw an exception going to the catch block
+			// (that is, fetching data on previous line should throw an exception due to the mock)
+			fail();
+		}
+		catch (Exception e)
+		{
+			e.printStackTrace();
+			assertTrue(e.getMessage().contains("Connection timed out. Number of retries (1) exceeded. No further attempts will be made."));
 		}
 	}
 }
\ No newline at end of file
diff --git a/src/test/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessorTest.java b/src/test/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessorTest.java
index 2d0b4f64..138ecde7 100644
--- a/src/test/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessorTest.java
+++ b/src/test/java/org/reactome/util/ensembl/EnsemblServiceResponseProcessorTest.java
@@ -6,16 +6,19 @@
 import static org.reactome.util.ensembl.EnsemblServiceResponseProcessor.MAX_TIMES_TO_WAIT;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
 import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.commons.lang.StringUtils;
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.StatusLine;
-import org.apache.http.entity.BasicHttpEntity;
-import org.apache.http.message.BasicHeader;
 import org.apache.logging.log4j.Logger;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -35,9 +38,8 @@ public class EnsemblServiceResponseProcessorTest {
 	private final String DUMMY_RESPONSE_CONTENT = "Dummy Content";
 
 	@Mock
-	private HttpResponse response;
-	@Mock
-	private StatusLine statusLine;
+	private HttpURLConnection urlConnection;
+
 	@Mock
 	private Logger logger;
 
@@ -54,10 +56,10 @@ public void createEnsemblServiceResponseProcessor() {
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterSingleResponseWithRetryAfter() {
+	public void correctEnsemblServiceResultAfterSingleResponseWithRetryAfter() throws IOException {
 		mockResponseWithRetryHeader(TOO_MANY_REQUESTS_STATUS_CODE, TOO_MANY_REQUESTS_REASON_PHRASE);
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWithRetryAfter(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWithRetryAfter(urlConnection);
 
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(TOO_MANY_REQUESTS_STATUS_CODE)));
 		assertThat("isOkayToRetry is false", result.isOkToRetry(), is(true));
@@ -67,18 +69,18 @@ public void correctEnsemblServiceResultAfterSingleResponseWithRetryAfter() {
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterMaximumNumberOfResponsesWithRetryAfter() {
+	public void correctEnsemblServiceResultAfterMaximumNumberOfResponsesWithRetryAfter() throws IOException {
 		mockResponseWithRetryHeader(TOO_MANY_REQUESTS_STATUS_CODE, TOO_MANY_REQUESTS_REASON_PHRASE);
 
 		// First call the method up to the maximum number of times
 		for (int i = 1; i < MAX_TIMES_TO_WAIT; i++) {
-			ensemblServiceResponseProcessor.processResponseWithRetryAfter(response);
+			ensemblServiceResponseProcessor.processResponseWithRetryAfter(urlConnection);
 		}
 
 		final int expectedMultiplierAfterMaximumNumberOfResponsesWithRetry = MAX_TIMES_TO_WAIT;
 		final int expectedWaitTime = RETRY_AFTER_VALUE * expectedMultiplierAfterMaximumNumberOfResponsesWithRetry;
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWithRetryAfter(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWithRetryAfter(urlConnection);
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(TOO_MANY_REQUESTS_STATUS_CODE)));
 		assertThat("isOkayToRetry is true", result.isOkToRetry(), is(false));
 		assertThat("Wait time is unexpected", result.getWaitTime(),
@@ -86,14 +88,14 @@ public void correctEnsemblServiceResultAfterMaximumNumberOfResponsesWithRetryAft
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterOkayResponse() {
-		final int okayStatusCode = HttpStatus.SC_OK;
+	public void correctEnsemblServiceResultAfterOkayResponse() throws IOException {
+		final int okayStatusCode = HttpURLConnection.HTTP_OK;
 		final String okayReasonPhrase = "OK";
 
 		mockResponse(okayStatusCode, okayReasonPhrase);
 		mockResponseEntityWithContent();
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(okayStatusCode)));
 		assertThat("Result content is unexpected", result.getResult(), is(equalTo(DUMMY_RESPONSE_CONTENT)));
@@ -102,13 +104,13 @@ public void correctEnsemblServiceResultAfterOkayResponse() {
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterGatewayTimeoutResponse() {
-		final int gatewayTimeoutStatusCode = HttpStatus.SC_GATEWAY_TIMEOUT;
+	public void correctEnsemblServiceResultAfterGatewayTimeoutResponse() throws IOException {
+		final int gatewayTimeoutStatusCode = HttpURLConnection.HTTP_GATEWAY_TIMEOUT;
 		final String gatewayTimeoutReasonPhrase = "Gateway Time-out";
 
 		mockResponse(gatewayTimeoutStatusCode, gatewayTimeoutReasonPhrase);
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(gatewayTimeoutStatusCode)));
 		assertThat("Non-empty result", result.getResult(), is(equalTo(StringUtils.EMPTY)));
@@ -117,8 +119,8 @@ public void correctEnsemblServiceResultAfterGatewayTimeoutResponse() {
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterGatewayTimeoutResponseAndMaximumTimeoutRetriesAttempted() {
-		final int gatewayTimeoutStatusCode = HttpStatus.SC_GATEWAY_TIMEOUT;
+	public void correctEnsemblServiceResultAfterGatewayTimeoutResponseAndMaximumTimeoutRetriesAttempted() throws IOException {
+		final int gatewayTimeoutStatusCode = HttpURLConnection.HTTP_GATEWAY_TIMEOUT;
 		final String gatewayTimeoutReasonPhrase = "Gateway Time-out";
 		final int allowedTimeoutRetries = ensemblServiceResponseProcessor.getTimeoutRetriesRemaining();
 
@@ -126,10 +128,10 @@ public void correctEnsemblServiceResultAfterGatewayTimeoutResponseAndMaximumTime
 
 		// Use all allowed timeout retries
 		for (int i = 1; i < allowedTimeoutRetries; i++) {
-			ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+			ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 		}
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(gatewayTimeoutStatusCode)));
 		assertThat("Non-empty result", result.getResult(), is(equalTo(StringUtils.EMPTY)));
 		assertThat("isOkayToRetry is true", result.isOkToRetry(), is(equalTo(false)));
@@ -140,13 +142,13 @@ public void correctEnsemblServiceResultAfterGatewayTimeoutResponseAndMaximumTime
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterNotFoundResponse() {
-		final int notFoundStatusCode = HttpStatus.SC_NOT_FOUND;
+	public void correctEnsemblServiceResultAfterNotFoundResponse() throws IOException {
+		final int notFoundStatusCode = HttpURLConnection.HTTP_NOT_FOUND;
 		final String notFoundReasonPhrase = "Not Found";
 
 		mockResponse(notFoundStatusCode, notFoundReasonPhrase);
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(notFoundStatusCode)));
 		assertThat("Non-empty result", result.getResult(), is(equalTo(StringUtils.EMPTY)));
@@ -155,13 +157,13 @@ public void correctEnsemblServiceResultAfterNotFoundResponse() {
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterInternalServerErrorResponse() {
-		final int internalServerErrorStatusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR;
+	public void correctEnsemblServiceResultAfterInternalServerErrorResponse() throws IOException {
+		final int internalServerErrorStatusCode = HttpURLConnection.HTTP_INTERNAL_ERROR;
 		final String internalServerErrorReasonPhrase = "Internal Server Error";
 
 		mockResponse(internalServerErrorStatusCode, internalServerErrorReasonPhrase);
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(internalServerErrorStatusCode)));
 		assertThat("Non-empty result", result.getResult(), is(equalTo(StringUtils.EMPTY)));
@@ -170,14 +172,14 @@ public void correctEnsemblServiceResultAfterInternalServerErrorResponse() {
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterBadRequestResponse() {
-		final int badRequestStatusCode = HttpStatus.SC_BAD_REQUEST;
+	public void correctEnsemblServiceResultAfterBadRequestResponse() throws IOException {
+		final int badRequestStatusCode = HttpURLConnection.HTTP_BAD_REQUEST;
 		final String badRequestReasonPhrase = "Bad Request";
 
 		mockResponse(badRequestStatusCode, badRequestReasonPhrase);
 		mockResponseEntityWithContent();
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(badRequestStatusCode)));
 		assertThat("Non-empty result", result.getResult(), is(equalTo(StringUtils.EMPTY)));
@@ -186,14 +188,14 @@ public void correctEnsemblServiceResultAfterBadRequestResponse() {
 	}
 
 	@Test
-	public void correctEnsemblServiceResultAfterUnexpectedResponse() {
-		final int movedPermanentlyStatusCode = HttpStatus.SC_MOVED_PERMANENTLY;
+	public void correctEnsemblServiceResultAfterUnexpectedResponse() throws IOException {
+		final int movedPermanentlyStatusCode = HttpURLConnection.HTTP_MOVED_PERM;
 		final String movedPermanentlyReasonPhrase = "Moved Permanently";
 
 		mockResponse(movedPermanentlyStatusCode, movedPermanentlyReasonPhrase);
 		mockResponseEntityWithContent();
 
-		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(response);
+		EnsemblServiceResult result = ensemblServiceResponseProcessor.processResponseWhenNotOverQueryQuota(urlConnection);
 
 		assertThat("Incorrect status code", result.getStatus(), is(equalTo(movedPermanentlyStatusCode)));
 		assertThat("Result content is unexpected", result.getResult(), is(equalTo(DUMMY_RESPONSE_CONTENT)));
@@ -202,54 +204,56 @@ public void correctEnsemblServiceResultAfterUnexpectedResponse() {
 	}
 
 	@Test
-	public void correctNumberOfRequestsRemainingSetAfterResponseWithXRateLimit() {
-		final int okayStatusCode = HttpStatus.SC_OK;
+	public void correctNumberOfRequestsRemainingSetAfterResponseWithXRateLimit() throws IOException {
+		final int okayStatusCode = HttpURLConnection.HTTP_OK;
 		final String okayReasonPhrase = "OK";
 
 		mockResponse(okayStatusCode, okayReasonPhrase);
 		mockXRateLimitHeader();
 
-		ensemblServiceResponseProcessor.processXRateLimitRemaining(response);
+		ensemblServiceResponseProcessor.processXRateLimitRemaining(urlConnection);
 
 		assertThat(
 			EnsemblServiceResponseProcessor.getNumRequestsRemaining(), is(equalTo(X_RATE_LIMIT_REMAINING_VALUE))
 		);
 	}
 
-	private void mockResponseWithRetryHeader(int statusCode, String reasonPhrase) {
+	private void mockResponseWithRetryHeader(int statusCode, String reasonPhrase) throws IOException {
 		mockResponse(statusCode, reasonPhrase);
 		mockRetryHeader();
 	}
 
-	private void mockResponse(int statusCode, String reasonPhrase) {
-		Mockito.when(response.getStatusLine()).thenReturn(statusLine);
-		Mockito.when(statusLine.getStatusCode()).thenReturn(statusCode);
-		Mockito.when(statusLine.getReasonPhrase()).thenReturn(reasonPhrase);
+	private void mockResponse(int statusCode, String reasonPhrase) throws IOException {
+		Mockito.when(urlConnection.getResponseMessage()).thenReturn("HTTP/1.0 " + statusCode + " " + reasonPhrase);
+		Mockito.when(urlConnection.getResponseCode()).thenReturn(statusCode);
 	}
 
 	private void mockRetryHeader() {
 		final String retryAfterHeaderName = "Retry-After";
 		final String retryAfterHeaderValue = Integer.toString(RETRY_AFTER_VALUE);
-		final Header[] headers = new Header[] {new BasicHeader(retryAfterHeaderName, retryAfterHeaderValue)};
-		Mockito.when(response.getAllHeaders()).thenReturn(headers);
-		Mockito.when(response.getHeaders(retryAfterHeaderName)).thenReturn(headers);
+		Map> headers = new HashMap<>();
+		headers.put(retryAfterHeaderName, Collections.singletonList(retryAfterHeaderValue));
+
+//		final Header[] headers = new Header[] {new BasicHeader(retryAfterHeaderName, retryAfterHeaderValue)};
+		Mockito.when(urlConnection.getHeaderFields()).thenReturn(headers);
 	}
 
 	private void mockXRateLimitHeader() {
 		final String xRateLimitHeaderName = "X-RateLimit-Remaining";
 		final String xRateLimitHeaderValue = Integer.toString(X_RATE_LIMIT_REMAINING_VALUE);
-		final Header[] headers = new Header[] {new BasicHeader(xRateLimitHeaderName, xRateLimitHeaderValue)};
-		Mockito.when(response.containsHeader(xRateLimitHeaderName)).thenReturn(true);
-		Mockito.when(response.getAllHeaders()).thenReturn(headers);
-		Mockito.when(response.getHeaders(xRateLimitHeaderName)).thenReturn(headers);
+		//final Header[] headers = new Header[] {new BasicHeader(xRateLimitHeaderName, xRateLimitHeaderValue)};
+
+		Map> headers = new HashMap<>();
+		headers.put(xRateLimitHeaderName, Collections.singletonList(xRateLimitHeaderValue));
+
+		Mockito.when(urlConnection.getHeaderFields()).thenReturn(headers);
 	}
 
-	private void mockResponseEntityWithContent() {
-		final BasicHttpEntity entity = new BasicHttpEntity();
+	private void mockResponseEntityWithContent() throws IOException {
 		final InputStream content = new ByteArrayInputStream(DUMMY_RESPONSE_CONTENT.getBytes());
 
-		entity.setContent(content);
-		entity.setContentLength(DUMMY_RESPONSE_CONTENT.getBytes().length);
-		Mockito.when(response.getEntity()).thenReturn(entity);
+		Mockito.when(urlConnection.getInputStream()).thenReturn(content);
+		Mockito.when(urlConnection.getContent()).thenReturn(content);
+		Mockito.when(urlConnection.getContentLength()).thenReturn(DUMMY_RESPONSE_CONTENT.getBytes().length);
 	}
 }
\ No newline at end of file