Switch over to .parquet files to back a Region2Vec dataset

.github/workflows/black.yml

@@ -1,11 +1,13 @@
 name: Lint
 
-on: [pull_request]
+on: 
+  pull_request:
+    branches: master
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: psf/black@20.8b1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - uses: psf/black@stable

.github/workflows/python-publish.yml

@@ -0,0 +1,31 @@
+  # This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    name: upload release to PyPI
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/run-pytest-dev.yml

@@ -0,0 +1,41 @@
+name: Run pytests for dev
+
+on:
+  pull_request:
+    branches: [dev]
+
+jobs:
+
+  pytest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Hack setup-python cache
+
+        if: hashFiles('**/requirements.txt', '**/pyproject.toml') == ''
+        run: |
+          touch ./requirements.txt
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'  # caching can speed up the workflow by reusing the installed dependencies
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Install test dependencies
+        run: if [ -f requirements/requirements-test.txt ]; then uv pip install -r requirements/requirements-test.txt --system; fi
+
+      - name: Install package
+        run: uv pip install .[ml] --system
+
+      - name: Run pytest tests
+        run: pytest tests -x -vv --remote-data

.github/workflows/run-pytest-release.yml

@@ -0,0 +1,35 @@
+name: Run pytests for release
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+
+  pytest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.12"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Install test dependencies
+        run: if [ -f requirements/requirements-test.txt ]; then uv pip install -r requirements/requirements-test.txt --system; fi
+
+      - name: Install package
+        run: uv pip install .[ml] --system
+
+      - name: Run pytest tests
+        run: pytest tests -x -vv --remote-data
+

.gitignore

@@ -85,7 +85,7 @@ ipython_config.py
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+.python-version
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -156,14 +156,14 @@ cython_debug/
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea name.
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-gitk/__pycache__
-gitk/eval/__pycache__
-gitk/region2vec/__pycache__
-gitk/tokenization/__pycache__
-gitk/tokenization/bedtools
+geniml/__pycache__
+geniml/eval/__pycache__
+geniml/region2vec/__pycache__
+geniml/tokenization/__pycache__
+geniml/tokenization/bedtools
 # for testing
 test.*
 
@@ -172,6 +172,30 @@ test.*
 # data
 tests/data/buenrostro2018.h5ad
 tests/data/buenrostro_metadata.tsv
+tests/data/model-tests/*
+bedshifted*
+examples/sh_output*
+examples/py_output*
 
 # integration test stuff
 tests/integration/buenrostro2018.model
+
+# examples
+examples/scembed/pbmc/
+examples/scembed/buenrostro
+examples/scembed/atlas
+examples/scembed/luecken2021
+
+# vector db stuff
+qdrant_storage/
+
+.ruff_cache/
+
+
+# MacOS
+.DS_Store
+
+local_cache
+
+lightning_logs
+data/

.readthedocs.yaml

@@ -0,0 +1,15 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+mkdocs:
+  configuration: mkdocs.yml
+  fail_on_warning: false
+
+# Optionally declare the Python requirements required to build your docs
+python:
+  install:
+  - requirements: requirements/requirements-doc.txt

.vscode/settings.json

@@ -1,11 +1,10 @@
 {
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
     "python.testing.pytestArgs": [
         "tests"
     ],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true,
-    "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
-    },
-    "python.formatting.provider": "none"
+    "python.testing.pytestEnabled": true
 }

LICENSE.txt

@@ -0,0 +1,9 @@
+Copyright 2023 Nathan Sheffield
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE.txt
+include requirements/*
+include pyproject.toml

README.md

@@ -1,3 +1,35 @@
-# Genomic interval toolkit
+# Genomic interval machine learning (geniml)
+
+Geniml is a python package for building machine learning models of genomic interval data (BED files). It also includes ancillary functions to support other types of analyses of genomic interval data.
+
+Documentation is hosted at <https://docs.bedbase.org/geniml/>.
+
+
+## Installation
+### To install `geniml` use this commands.
+
+Without specifying dependencies, the default dependencies will be installed, 
+which DO NOT include machine learning (ML) or heavy processing libraries.
+
+
+From pypi:
+```
+pip install geniml
+```
+or install the latest version from the GitHub repository:
+```
+pip install git+https://github.com/databio/geniml.git
+```
+
+### To install Machine learning dependencies use this command:
+
+From pypi:
+```
+pip install geniml[ml]
+```
+
+
+## Development
+
+Run tests (from `/tests`) with `pytest`. Please read the [contributor guide](https://docs.bedbase.org/geniml/contributing/) to contribute.
 
-You can find documentation in the `docs` subfolder.

docs/README.md

@@ -1,41 +1,8 @@
-# Genomic interval toolkit
+# Geniml documentation
 
-## Introduction
+The documentation for `geniml` is now part of BEDbase. You can find
 
-`gitk` is a suite of tools for apply machine learning approaches to genomic interval data. It is organized as a set of modules that provide related functions, such as building HMMs, assessing genomic interval universes, calculating likelihoods of consensus genomic interval sets, and computing single-cell clusters.
-
-## Install
-
-```
-pip install --user --upgrade .
-```
-
-## gitk modules
-
-- [gitk/hmm](gitk/hmm) - Building HMMs
-- [gitk/assess](gitk/assess) - Assess universe fit
-- [gitk/likelihood](gitk/likelihood) - Calculate likelihood of universe
-- [gitk/scembed](gitk/scembed) - Compute single-cell clusters from a cell-feature matrix using Word2Vec
-
-## Using modules from Python
-
-This repo is divided into modules. Each module should be written in a way that it provides utility as a Python library. For example, you can call functions in the `hmm` module like this:
-
-```
-import gitk
-
-gitk.hmm.function()
-```
-
-## Command-line interfaces
-
-In addition to being importable from Python, *some* modules also provide a CLI. For these, developers provide a subcommand for CLI use. The root `gitk` package provides a generalized command-line interface with the command `gitk`. The modules that provide CLIs then correspond to CLI commands, *e.g* `gitk hmm` or `gitk likelihood`, with the corresponding code contained within a sub-folder named after the model:
-
-```
-gitk <module> ...
-```
-
-This is implemented within each module folder with:
-
-- `gitk/<module>/cli.py` - defines the command-line interface and provides a subparser for this module's CLI command.
+- the rendered [documentation for geniml](https://docs.bedbase.org/geniml/).
+- the [repository with the documentation source](https://github.com/databio/bedbase). 
 
+If you have any questions, please open an issue on [this repository](https://github.com/databio/geniml/issues) or on the [bedbase](https://github.com/databio/bedbase/issues) repository.