Merge pull request #98 from nextstrain/improve-ppx-fetch

ingest: Improve fetch from PPX

Author: joverlee521

ingest/config/config.yaml

@@ -9,10 +9,10 @@ ncbi_taxon_id:
 
 ppx_fetch:
   a:
-    seqs: https://lapis.pathoplexus.org/rsv-a/sample/unalignedNucleotideSequences
+    seqs: https://lapis.pathoplexus.org/rsv-a/sample/unalignedNucleotideSequences?versionStatus=LATEST_VERSION
     meta: https://lapis.pathoplexus.org/rsv-a/sample/details?dataFormat=csv&versionStatus=LATEST_VERSION
   b:
-    seqs: https://lapis.pathoplexus.org/rsv-b/sample/unalignedNucleotideSequences
+    seqs: https://lapis.pathoplexus.org/rsv-b/sample/unalignedNucleotideSequences?versionStatus=LATEST_VERSION
     meta: https://lapis.pathoplexus.org/rsv-b/sample/details?dataFormat=csv&versionStatus=LATEST_VERSION
 
 

ingest/workflow/snakemake_rules/fetch_sequences.smk

@@ -102,9 +102,11 @@ rule download_ppx_seqs:
         sequences= "data/{subtype}/ppx_sequences.fasta",
     params:
         sequences_url=lambda w: config["ppx_fetch"][w.subtype]["seqs"],
+    # Allow retries in case of network errors
+    retries: 5
     shell:
         """
-        curl  {params.sequences_url:q} -o {output.sequences}
+        curl -fsSL {params.sequences_url:q} -o {output.sequences}
         """
 
 rule download_ppx_meta:
@@ -113,9 +115,11 @@ rule download_ppx_meta:
     params:
         metadata_url=lambda w: config["ppx_fetch"][w.subtype]["meta"],
         fields = ",".join(config["ppx_metadata_fields"])
+    # Allow retries in case of network errors
+    retries: 5
     shell:
         """
-        curl  '{params.metadata_url}&fields={params.fields}' -o {output.metadata}
+        curl -fsSL '{params.metadata_url}&fields={params.fields}' -o {output.metadata}
         """
 
 rule format_ppx_ndjson: