From 008957e576af268af38392358b70900323651469 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Tue, 22 Mar 2022 22:54:57 -0400 Subject: [PATCH 01/15] add Dockerfile for test0.0.0 --- Dockerfile | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3d55487 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,84 @@ +# Building Docker file for ModelArray (R package) + ConFixel (python package). Core is config.yml for ModelArray's circle.ci. +# When update this Dockerfile, please update: +# 1. rocker/verse: +# 2. commitSHA_confixel +# 3. commitSHA_modelarray - see towards the end of this file +# 4. ModelArray's dependent R packages - see DESCRIPTION file + +## Base image https://hub.docker.com/u/rocker/ +FROM rocker/verse:4.1.2 + +## versions and parameters: +# specify the commit SHA: # e.g. https://github.com/PennLINC/qsiprep/blob/master/Dockerfile#L174 +ENV commitSHA_confixel="eda00bc1c9837b5586a65f24b4c986845f41582c" + + +RUN mkdir /home/data +# RUN mkdir /home/ModelArray + +## Install libraries +# ref: .circleci/config.yml from ModelArray: +RUN apt-get update && apt-get install -y --no-install-recommends \ + libhdf5-dev \ + texlive-fonts-recommended \ + git + +# # Install libraries +# RUN apt-get update && apt-get install -y --no-install-recommends libhdf5-dev +# # Install libraries for latex +# RUN apt-get update && apt-get install -y --no-install-recommends texlive-fonts-recommended +# Install git: # ref a bit from: https://github.com/PennLINC/qsiprep_build/blob/main/Dockerfile_DSIStudio +# RUN apt-get update && apt-get install -y --no-install-recommends git + +# Install python: # ref: https://github.com/PennLINC/flaudit/blob/master/Dockerfile#L23 +RUN apt-get update && apt-get install -y python3-pip python3-dev + + +## Install ConFixel (python package) +# git clone xxxx.git${commitSHA} +# cd to directory +# pip install . + +RUN git clone -n https://github.com/PennLINC/ConFixel.git +WORKDIR ConFixel +RUN git checkout ${commitSHA_confixel} +RUN pip install . +WORKDIR / +# RUN rm -r ConFixel + + +## Install dependent R packages: +# from CRAN: # removed base packages from the list (otherwise warning in docker build): methods and parallel +RUN install2.r --error --ncpus -4 \ + matrixStats \ + magrittr \ + dplyr \ + tidyr \ + tibble \ + stringr \ + glue \ + doParallel \ + hdf5r \ + mgcv \ + rlang \ + broom \ + pbmcapply \ + pbapply \ + crayon + +# from Bioc: # first, install BiocManager from CRAN: +RUN install2.r --error BiocManager +RUN R -e 'BiocManager::install("HDF5Array")' +RUN R -e 'BiocManager::install("rhdf5")' +RUN R -e 'BiocManager::install("DelayedArray")' + +# # copy necessary files and folders: +# COPY ./install_packages.R ./install_packages.R +# # install R-packages: +# RUN Rscript ./install_packages.R + + +## install ModelArray (R package) +# please update commit SHA for ModelArray if needed +RUN R -e 'devtools::install_github("PennLINC/ModelArray@9e735b93f2d6b756f8ef18aadc14e2d10c6cc191")' + From 784055800a79da58feb5a31c7eb83116a2dc77cc Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Tue, 29 Mar 2022 16:26:27 -0400 Subject: [PATCH 02/15] include R 4.1.0 in circle ci tests --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b60ccf..6583330 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,10 +47,10 @@ workflows: version: 2 build_test_deploy: jobs: - #- releaseR4.1.0: # temporarily comment out as there is error with matrixStats package - # filters: - # tags: - # only: /.*/ + - releaseR4.1.0: + filters: + tags: + only: /.*/ #- oldR3.5.3: # filters: # tags: From ef0de212c038d37049973cfeb49052c9849b3385 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Tue, 29 Mar 2022 17:52:36 -0400 Subject: [PATCH 03/15] try matrixStats version >= 0.61.0 for circle ci R version 4.1.0 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6c6b610..c7a6ed4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,7 +14,7 @@ LazyData: true Depends: R (>= 4.1.0) biocViews: Imports: - matrixStats (>= 0.60.1), + matrixStats (>= 0.61.0), magrittr, methods, dplyr, From 2470c78309887aae54e40b27cf83ff0f34dbfc98 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Tue, 29 Mar 2022 18:03:45 -0400 Subject: [PATCH 04/15] directly install MatrixGenerics that DelayedArray depends on --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index c7a6ed4..5f3fa90 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,7 +14,7 @@ LazyData: true Depends: R (>= 4.1.0) biocViews: Imports: - matrixStats (>= 0.61.0), + MatrixGenerics, magrittr, methods, dplyr, From 9a08ff7f0b7adbc1b91fd99c6cc77103741fd042 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Tue, 29 Mar 2022 18:17:13 -0400 Subject: [PATCH 05/15] include both matrixStats and MatrixGenerics in DESCRIPTION file --- DESCRIPTION | 1 + 1 file changed, 1 insertion(+) diff --git a/DESCRIPTION b/DESCRIPTION index 5f3fa90..526ea31 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,6 +14,7 @@ LazyData: true Depends: R (>= 4.1.0) biocViews: Imports: + matrixStats (>= 0.61.0), MatrixGenerics, magrittr, methods, From b834417aceaddaaf1b3fd5c58bf3962e0c30db95 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Wed, 30 Mar 2022 10:10:35 -0400 Subject: [PATCH 06/15] delete matrixStats and MatrixGenerics in the DESCRIPTION list; also delete the comma at the end --- DESCRIPTION | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 526ea31..ba7e32f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,8 +14,6 @@ LazyData: true Depends: R (>= 4.1.0) biocViews: Imports: - matrixStats (>= 0.61.0), - MatrixGenerics, magrittr, methods, dplyr, @@ -34,7 +32,7 @@ Imports: broom, pbmcapply, pbapply, - crayon, + crayon RoxygenNote: 7.1.2 Suggests: rmarkdown, From 576912b4f0d06b6a88a89e9edacc94f002d40428 Mon Sep 17 00:00:00 2001 From: zhao-cy Date: Wed, 30 Mar 2022 12:19:53 -0400 Subject: [PATCH 07/15] test R version 3.6.0; also update required R version in DESCRIPTION --- .circleci/config.yml | 10 +++++++++- DESCRIPTION | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6583330..4ef125f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,6 +34,10 @@ jobs: docker: - image: rocker/verse:4.1.0 <<: *steps + releaseR3.6.0: + docker: + - image: rocker/verse:3.6.0 + <<: *steps oldR3.5.3: docker: - image: rocker/verse:3.5.3 @@ -47,7 +51,11 @@ workflows: version: 2 build_test_deploy: jobs: - - releaseR4.1.0: + #- releaseR4.1.0: + # filters: + # tags: + # only: /.*/ + - releaseR3.6.0: filters: tags: only: /.*/ diff --git a/DESCRIPTION b/DESCRIPTION index ba7e32f..0631cb7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -11,7 +11,7 @@ Description: What the package does (one paragraph). License: BSD 3-Clause License + file LICENSE Encoding: UTF-8 LazyData: true -Depends: R (>= 4.1.0) +Depends: R (>= 3.6.0) biocViews: Imports: magrittr, From 1cc5fde4bcb3b513bef2f1e80d28130202a7b0e1 Mon Sep 17 00:00:00 2001 From: zhao-cy Date: Wed, 30 Mar 2022 12:50:58 -0400 Subject: [PATCH 08/15] try R version 3.6.3; also updated required R version in DESCRIPTION --- .circleci/config.yml | 6 +++--- DESCRIPTION | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ef125f..a14dba6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,9 +34,9 @@ jobs: docker: - image: rocker/verse:4.1.0 <<: *steps - releaseR3.6.0: + releaseR3.6.3: docker: - - image: rocker/verse:3.6.0 + - image: rocker/verse:3.6.3 <<: *steps oldR3.5.3: docker: @@ -55,7 +55,7 @@ workflows: # filters: # tags: # only: /.*/ - - releaseR3.6.0: + - releaseR3.6.3: filters: tags: only: /.*/ diff --git a/DESCRIPTION b/DESCRIPTION index 0631cb7..1c9a9a3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -11,7 +11,7 @@ Description: What the package does (one paragraph). License: BSD 3-Clause License + file LICENSE Encoding: UTF-8 LazyData: true -Depends: R (>= 3.6.0) +Depends: R (>= 3.6.3) biocViews: Imports: magrittr, From b5cfe0165dfe8ba735ee1759294c62c559061722 Mon Sep 17 00:00:00 2001 From: zhao-cy Date: Wed, 30 Mar 2022 17:38:06 -0400 Subject: [PATCH 09/15] added as.character() as R 3.6.3 will take phenotypes[[source_file]] as factors instead of characters which leads to error of 'matching source file names were not successful' --- R/analyse.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/R/analyse.R b/R/analyse.R index 4d91690..649d1ae 100644 --- a/R/analyse.R +++ b/R/analyse.R @@ -67,7 +67,7 @@ ModelArray.lm <- function(formula, data, phenotypes, scalar, element.subset = NU ### sanity check: whether they match: modelarray's source file list and phenotypes' source file list: sources.modelarray <- sources(data)[[scalar]] - sources.phenotypes <- phenotypes[["source_file"]] + sources.phenotypes <- phenotypes[["source_file"]] %>% as.character() if (is.null(sources.phenotypes)) { stop(paste0("Did not find column 'source_file' in argument 'phenotypes'. Please check!")) } @@ -96,7 +96,7 @@ ModelArray.lm <- function(formula, data, phenotypes, scalar, element.subset = NU # apply to phenotypes: phenotypes <- phenotypes[reorder_idx, ] row.names(phenotypes) <- NULL # reset the row name, just to be safe for later adding scalar values... see ModelArray_paper/notebooks/test_match_sourceFiles.Rmd - if (!identical(phenotypes[["source_file"]], sources.modelarray)) { + if (!identical(phenotypes[["source_file"]] %>% as.character(), sources.modelarray)) { stop("matching source file names were not successful...") } } else { @@ -454,7 +454,7 @@ ModelArray.gam <- function(formula, data, phenotypes, scalar, element.subset = N ### sanity check: whether they match: modelarray's source file list and phenotypes' source file list: sources.modelarray <- sources(data)[[scalar]] - sources.phenotypes <- phenotypes[["source_file"]] + sources.phenotypes <- phenotypes[["source_file"]] %>% as.character() if (is.null(sources.phenotypes)) { stop(paste0("Did not find column 'source_file' in argument 'phenotypes'. Please check!")) } @@ -483,7 +483,7 @@ ModelArray.gam <- function(formula, data, phenotypes, scalar, element.subset = N # apply to phenotypes: phenotypes <- phenotypes[reorder_idx, ] row.names(phenotypes) <- NULL # reset the row name, just to be safe for later adding scalar values... see ModelArray_paper/notebooks/test_match_sourceFiles.Rmd - if (!identical(phenotypes[["source_file"]], sources.modelarray)) { + if (!identical(phenotypes[["source_file"]] %>% as.character(), sources.modelarray)) { stop("matching source file names were not successful...") } } else { From 10eb12f3d6ff892ef65d8fda8529c5da980b7c2d Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Mon, 4 Apr 2022 15:56:07 -0400 Subject: [PATCH 10/15] add package version requirements for R 3.6.3 --- DESCRIPTION | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index ba7e32f..8957366 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,10 +14,13 @@ LazyData: true Depends: R (>= 4.1.0) biocViews: Imports: + vctrs (>= 0.3.0), + rlang (>= 0.4.6), + tidyselect (>= 1.1.0), magrittr, methods, dplyr, - tidyr, + tidyr (>= 1.1.0), tibble, stringr, glue, @@ -27,7 +30,6 @@ Imports: rhdf5, hdf5r, mgcv, - rlang, DelayedArray, broom, pbmcapply, From 24d6d6885345af4cb3e3e70e546140f44b9c6e77 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Mon, 4 Apr 2022 16:57:28 -0400 Subject: [PATCH 11/15] install specific vctrs package in docker ahead of install with DESCRIPTION --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a14dba6..1c712af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,6 +18,9 @@ steps: &steps - run: name: Install package dependencies (hdf5r) command: R -e "require(hdf5r)" # "lapply(c('hdf5r','HDF5Array','DelayedArray'), require, character.only=TRUE)" + - run: + name: Install R package dependencies (vctrs 0.3.0) + command: R -e "install.packages('https://cran.r-project.org/src/contrib/Archive/vctrs/vctrs_0.3.0.tar.gz', repo=NULL, type='source')" - run: name: Build package command: R CMD build . From 2cdecefb6a7f3cc7a19e9e328d36fac17d35c50e Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Mon, 4 Apr 2022 17:13:42 -0400 Subject: [PATCH 12/15] change the order of install dependent R packages in docker image --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c712af..7864eae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,14 +13,14 @@ steps: &steps name: Install libraries for latex command: apt-get update && apt-get install -y --no-install-recommends texlive-fonts-recommended - run: - name: Install package dependencies + name: Install R package dependencies (vctrs 0.3.0) + command: R -e "install.packages('https://cran.r-project.org/src/contrib/Archive/vctrs/vctrs_0.3.0.tar.gz', repo=NULL, type='source')" + - run: + name: Install package dependencies (via devtools::install_deps) command: R -e "devtools::install_deps(dep = TRUE)" - run: name: Install package dependencies (hdf5r) command: R -e "require(hdf5r)" # "lapply(c('hdf5r','HDF5Array','DelayedArray'), require, character.only=TRUE)" - - run: - name: Install R package dependencies (vctrs 0.3.0) - command: R -e "install.packages('https://cran.r-project.org/src/contrib/Archive/vctrs/vctrs_0.3.0.tar.gz', repo=NULL, type='source')" - run: name: Build package command: R CMD build . From 2e1bb19b36355e22ef1fd089f173b4175ed52719 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Mon, 4 Apr 2022 17:29:05 -0400 Subject: [PATCH 13/15] add upgrade='always' when devtools::install_deps() --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7864eae..c30a948 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,12 +12,9 @@ steps: &steps - run: name: Install libraries for latex command: apt-get update && apt-get install -y --no-install-recommends texlive-fonts-recommended - - run: - name: Install R package dependencies (vctrs 0.3.0) - command: R -e "install.packages('https://cran.r-project.org/src/contrib/Archive/vctrs/vctrs_0.3.0.tar.gz', repo=NULL, type='source')" - run: name: Install package dependencies (via devtools::install_deps) - command: R -e "devtools::install_deps(dep = TRUE)" + command: R -e "devtools::install_deps(dep = TRUE, upgrade = 'always')" - run: name: Install package dependencies (hdf5r) command: R -e "require(hdf5r)" # "lapply(c('hdf5r','HDF5Array','DelayedArray'), require, character.only=TRUE)" From 1dc1527dc378397a466325774724ba98c8741de8 Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Mon, 4 Apr 2022 18:12:26 -0400 Subject: [PATCH 14/15] only use install.packages() to install vctrs without specify .tar.gz with version number --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c30a948..aefd596 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,12 @@ steps: &steps - run: name: Install libraries for latex command: apt-get update && apt-get install -y --no-install-recommends texlive-fonts-recommended + - run: + name: Install R package dependencies (vctrs) + command: R -e "install.packages('vctrs')" + - run: + name: Check package version of vctrs + command: R -e "packageVersion('vctrs')" - run: name: Install package dependencies (via devtools::install_deps) command: R -e "devtools::install_deps(dep = TRUE, upgrade = 'always')" From 6f4684cf7dc608e3b3b2a862a6bc0f7db5d1808f Mon Sep 17 00:00:00 2001 From: Chenying Zhao Date: Tue, 5 Apr 2022 10:07:06 -0400 Subject: [PATCH 15/15] remove cache --- .circleci/config.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aefd596..49267d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,9 +2,6 @@ version: 2 steps: &steps steps: - - restore_cache: - keys: - - cache-{{ .Environment.CIRCLE_JOB }} - checkout - run: name: Install libraries @@ -13,14 +10,11 @@ steps: &steps name: Install libraries for latex command: apt-get update && apt-get install -y --no-install-recommends texlive-fonts-recommended - run: - name: Install R package dependencies (vctrs) - command: R -e "install.packages('vctrs')" + name: Install package dependencies (via devtools::install_deps) + command: R -e "devtools::install_deps(dep = TRUE, upgrade = 'always')" - run: name: Check package version of vctrs command: R -e "packageVersion('vctrs')" - - run: - name: Install package dependencies (via devtools::install_deps) - command: R -e "devtools::install_deps(dep = TRUE, upgrade = 'always')" - run: name: Install package dependencies (hdf5r) command: R -e "require(hdf5r)" # "lapply(c('hdf5r','HDF5Array','DelayedArray'), require, character.only=TRUE)" @@ -30,10 +24,6 @@ steps: &steps - run: name: Check package command: R CMD check *tar.gz - - save_cache: - key: cache-{{ .Environment.CIRCLE_JOB }} - paths: - - "/usr/local/lib/R/site-library" jobs: releaseR4.1.0: