v0.0.2

.github/workflows/black.yml

@@ -0,0 +1,11 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: psf/black@stable

.github/workflows/test-pipeline.yml

@@ -0,0 +1,70 @@
+name: Test bedstat pipeline
+
+on:
+  push:
+    branches: [master, dev]
+  pull_request:
+    branches: [master, dev]
+
+jobs:
+  pytest:
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        os: [ubuntu-latest] # can't use macOS when using service containers or container jobs
+        r: [release]
+    runs-on: ${{ matrix.os }}
+    services:
+        postgres:
+          image: postgres
+          env:
+            POSTGRES_USER: postgres
+            POSTGRES_PASSWORD: bedbasepassword
+            POSTGRES_DB: postgres
+          ports:
+          - 5432:5432
+          options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependancies
+      run: if [ -f requirements/requirements-all.txt ]; then pip install -r requirements/requirements-all.txt; fi
+
+    - name: Install dev dependancies
+      run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi
+
+    - name: Install test dependancies
+      run: if [ -f requirements/requirements-test.txt ]; then pip install -r requirements/requirements-test.txt; fi
+
+    - name: Install libcurl
+      run: |
+        sudo apt-get update
+        sudo apt-get install libcurl4-openssl-dev
+
+    - uses: r-lib/actions/setup-r@master
+      with:
+        r-version: ${{ matrix.r }}
+
+    - name: Install R dependancies
+      run: Rscript scripts/installRdeps.R
+
+    - name: Run pipeline
+      run: looper run -p local tests/data/bedstat_config.yaml
+
+    - name: Test plots exist
+      run: |
+        if ls $GITHUB_WORKSPACE/outputs/bedstat_output/a6a08126cb6f4b1953ba0ec8675df85a/test_hg38*.png 1> /dev/null 2>&1; then
+          echo "SUCCESS";
+        else
+          echo "ERROR: files do not exist: $GITHUB_WORKSPACE/outputs/bedstat_output/a6a08126cb6f4b1953ba0ec8675df85a/test_hg38*.png";
+          exit 1
+        fi
+
+    - name: Test record in PostgreSQL
+      run: |
+        echo "from bbconf import BedBaseConf; from bbconf.const import *; bbc = BedBaseConf('$GITHUB_WORKSPACE/tests/data/config_min.yaml'); assert bbc.bed.record_count == 1, 'Number of records in the bedfiles table not equal 1'" | python3 -

README.md

@@ -1,3 +1,4 @@
+![Test bedstat pipeline](https://github.com/databio/bedstat/workflows/Test%20bedstat%20pipeline/badge.svg)
 # bedstat
 pipeline for obtaining statistics about bed files
 
@@ -35,60 +36,37 @@ The input PEP can be validated against the [JSON schema in this repository](pep_
 eido validate <path/to/pep> -s https://schema.databio.org/pipelines/bedstat.yaml
 ```
 
-### 2. Create a persistent volume to house elasticsearch data
+### 2. Run PostgreSQL
 
-```
-docker volume create es-data
-```
-
-### 3. Run the docker container for elasticsearch
+For example, to run an instance in a container and make the data persist, execute:
 
 ```
-docker run -p 9200:9200 -p 9300:9300 -v es-data:/usr/share/elasticsearch/data -e "xpack.ml.enabled=false" \
-  -e "discovery.type=single-node" elasticsearch:7.5.1
+docker volume create postgres-data
+docker run -d --name bedbase-postgres -p 5432:5432 -e POSTGRES_PASSWORD=bedbasepassword -e POSTGRES_USER=postgres -e POSTGRES_DB=postgres -v postgres-data:/var/lib/postgresql/data postgres
 ```
+Provided environment variables need to match the settings in bedbase configuration file
 
-### 4. Run the bedstat pipeline on the PEP
+### 3. Run the bedstat pipeline on the PEP
 
-Then simply run the looper command to run the pipeline for each bed file. It will create a set of plots and statistics per bed file and insert the metadata into Elastic:
+Then simply run the `looper run` command to run the pipeline for each bed file. It will create a set of plots and statistics per bed file and insert the metadata into PostgreSQL:
 
 ```
 looper run project/bedstat_config.yaml
 ```
 
-The data loaded into elasticsearch should persist between elasticsearch invocations, on the es-data docker volume created above in step 2.
-
-### 5. (optional) Run Kibana
-
-Kibana can be used in order to see ElasticSearch data in a "GUI" kind of a way.
-
-Pull a matching Kibana docker image. Make sure the Elasticsearch and Kibana container tags match:
-```
-docker pull docker.elastic.co/kibana/kibana:7.5.1
-```
-
-Get the ID of the docker container (started above) running ElasticSearch via 
-```
-docker ps | grep elasticsearch
-```
-
-Run Kibana to link to that container:
-```
-docker run --link <ID OF ELASTIC CONTAINER HERE>:elasticsearch -p 5601:5601  docker.elastic.co/kibana/kibana:7.5.1
-```
-
-Point your local web browser to http://localhost:5601
-
----
+The data loaded into PostgreSQL should persist between PostgreSQL invocations, on the `postgres-data` docker volume created above in step 2.
 
 ## Additional dependencies
 
 [regionstat.R](tools/regionstat.R) script is used to calculate the bed file statistics, so the pipeline also depends on several R packages:
 
+* `R.utils`
 * `BiocManager`
 * `optparse`
 * `devtools`
 * `GenomicRanges`
+* `GenomicFeatures`
+* `ensembldb`
 * `GenomicDistributions`
 * `BSgenome.<organim>.UCSC.<genome>` *depending on the genome used* 
 * `LOLA`

pep_schema.yaml

@@ -18,7 +18,6 @@ properties:
         genome:
           type: string
           description: "organism genome code"
-          enum: ["hg18", "hg19", "hg38", "mm9", "mm10"]
         narrowpeak:
           type: integer
           minimum: 0