Jupyter & Language server support

.gitignore

@@ -64,3 +64,5 @@ dox
 # Testing playground #
 ######################
 scratch.seq
+
+.venv

.travis.yml

@@ -94,7 +94,7 @@ install:
   # install htslib:
   - wget -c https://github.com/samtools/htslib/releases/download/1.9/htslib-1.9.tar.bz2 -O - | tar -xj
   - cd htslib-1.9
-  - ./configure CFLAGS="-fPIC" --disable-libcurl --without-curses
+  - ./configure CFLAGS="-fPIC" --disable-libcurl
   - make
   - sudo make install
   - cd -

compiler/include/seq/expr.h

@@ -541,6 +541,7 @@ class MatchExpr : public Expr {
 class ConstructExpr : public Expr {
 private:
   mutable types::Type *type;
+  mutable types::Type *type0; // type before deduction, saved for clone
   std::vector<Expr *> args;
 
 public:
@@ -578,6 +579,18 @@ class OptExpr : public Expr {
   OptExpr *clone(Generic *ref) override;
 };
 
+class YieldExpr : public Expr {
+private:
+  Func *base;
+
+public:
+  YieldExpr(Func *base);
+  void resolveTypes() override;
+  llvm::Value *codegen0(BaseFunc *base, llvm::BasicBlock *&block) override;
+  types::Type *getType0() const override;
+  YieldExpr *clone(Generic *ref) override;
+};
+
 class DefaultExpr : public Expr {
 public:
   explicit DefaultExpr(types::Type *type);

compiler/include/seq/func.h

@@ -162,6 +162,7 @@ class Func : public BaseFunc, public Generic, public SrcObject {
   void codegenYield(llvm::Value *val, types::Type *type,
                     llvm::BasicBlock *&block, bool empty = false,
                     bool dryrun = false);
+  llvm::Value *codegenYieldExpr(llvm::BasicBlock *&block);
 
   bool isGen() override;
   Var *getArgVar(std::string name);

compiler/include/seq/funct.h

@@ -54,7 +54,9 @@ class GenType : public Type {
   llvm::Value *done(llvm::Value *self, llvm::BasicBlock *block);
   void resume(llvm::Value *self, llvm::BasicBlock *block,
               llvm::BasicBlock *normal, llvm::BasicBlock *unwind);
-  llvm::Value *promise(llvm::Value *self, llvm::BasicBlock *block);
+  llvm::Value *promise(llvm::Value *self, llvm::BasicBlock *block,
+                       bool returnPtr = false);
+  void send(llvm::Value *self, llvm::Value *val, llvm::BasicBlock *block);
   void destroy(llvm::Value *self, llvm::BasicBlock *block);
   bool fromPrefetch();
 

compiler/include/seq/num.h

@@ -98,6 +98,10 @@ class ByteType : public Type {
   llvm::Type *getLLVMType(llvm::LLVMContext &context) const override;
   size_t size(llvm::Module *module) const override;
   static ByteType *get() noexcept;
+  /// returns ASCII complement table
+  static llvm::GlobalVariable *
+  getByteCompTable(llvm::Module *module,
+                   const std::string &name = "seq.byte_comp_table");
 };
 
 } // namespace types

compiler/lang/expr.cpp

@@ -1943,7 +1943,7 @@ MatchExpr *MatchExpr::clone(Generic *ref) {
 }
 
 ConstructExpr::ConstructExpr(types::Type *type, std::vector<Expr *> args)
-    : Expr(), type(type), args(std::move(args)) {}
+    : Expr(), type(type), type0(nullptr), args(std::move(args)) {}
 
 types::Type *ConstructExpr::getConstructType() { return type; }
 
@@ -2035,8 +2035,10 @@ types::Type *ConstructExpr::getType0() const {
 
   // type parameter deduction if constructing generic class:
   auto *ref = dynamic_cast<types::RefType *>(type);
-  if (ref && ref->numGenerics() > 0 && !ref->realized())
+  if (ref && ref->numGenerics() > 0 && !ref->realized()) {
+    type0 = type;
     type = ref->realize(ref->deduceTypesFromArgTypes(types));
+  }
 
   types::Type *ret = type->magicOut("__init__", types);
   return ret->is(types::Void) ? type : ret;
@@ -2046,7 +2048,8 @@ ConstructExpr *ConstructExpr::clone(Generic *ref) {
   std::vector<Expr *> argsCloned;
   for (auto *arg : args)
     argsCloned.push_back(arg->clone(ref));
-  SEQ_RETURN_CLONE(new ConstructExpr(type->clone(ref), argsCloned));
+  SEQ_RETURN_CLONE(
+      new ConstructExpr((type0 ? type0 : type)->clone(ref), argsCloned));
 }
 
 MethodExpr::MethodExpr(Expr *self, Func *func)
@@ -2089,6 +2092,26 @@ OptExpr *OptExpr::clone(Generic *ref) {
   SEQ_RETURN_CLONE(new OptExpr(val->clone(ref)));
 }
 
+YieldExpr::YieldExpr(Func *base) : Expr(), base(base) {}
+
+void YieldExpr::resolveTypes() { base->resolveTypes(); }
+
+Value *YieldExpr::codegen0(BaseFunc *base, BasicBlock *&block) {
+  assert(dynamic_cast<Func *>(base) == this->base);
+  return this->base->codegenYieldExpr(block);
+}
+
+types::Type *YieldExpr::getType0() const {
+  types::GenType *gen = base->getFuncType()->getBaseType(0)->asGen();
+  if (!gen)
+    throw exc::SeqException("yield expression in non-generator");
+  return gen->getBaseType(0);
+}
+
+YieldExpr *YieldExpr::clone(Generic *ref) {
+  SEQ_RETURN_CLONE(new YieldExpr(base->clone(ref)));
+}
+
 DefaultExpr::DefaultExpr(types::Type *type) : Expr(type) {}
 
 Value *DefaultExpr::codegen0(BaseFunc *base, BasicBlock *&block) {

compiler/lang/func.cpp

@@ -596,6 +596,27 @@ void Func::codegenYield(Value *val, types::Type *type, BasicBlock *&block,
   }
 }
 
+Value *Func::codegenYieldExpr(BasicBlock *&block) {
+  if (!gen)
+    throw exc::SeqException("yield expression in non-generator");
+
+  LLVMContext &context = block->getContext();
+  Function *suspFn = Intrinsic::getDeclaration(module, Intrinsic::coro_suspend);
+  Value *tok = ConstantTokenNone::get(context);
+  Value *final = ConstantInt::get(IntegerType::getInt1Ty(context), 0);
+  IRBuilder<> builder(block);
+  Value *susp = builder.CreateCall(suspFn, {tok, final});
+
+  block = BasicBlock::Create(block->getContext(), "", block->getParent());
+
+  SwitchInst *inst = builder.CreateSwitch(susp, suspend, 2);
+  inst->addCase(ConstantInt::get(IntegerType::getInt8Ty(context), 0), block);
+  inst->addCase(ConstantInt::get(IntegerType::getInt8Ty(context), 1), cleanup);
+
+  builder.SetInsertPoint(block);
+  return builder.CreateLoad(promise);
+}
+
 bool Func::isGen() { return yield != nullptr; }
 
 std::vector<std::string> Func::getArgNames() { return argNames; }

compiler/lang/patterns.cpp

@@ -312,6 +312,29 @@ void SeqPattern::resolveTypes(types::Type *type) {
         getSrcInfo());
 }
 
+// Returns the appropriate character for the given logical index, respecting
+// reverse complementation. Given `lenActual` should be non-negative.
+static Value *indexIntoSeq(Value *ptr, Value *lenActual, Value *rc, Value *idx,
+                           BasicBlock *block) {
+  LLVMContext &context = block->getContext();
+  IRBuilder<> builder(block);
+  Value *backIdx = builder.CreateSub(lenActual, idx);
+  backIdx = builder.CreateSub(backIdx, oneLLVM(context));
+
+  Value *charFwdPtr = builder.CreateGEP(ptr, idx);
+  Value *charRevPtr = builder.CreateGEP(ptr, backIdx);
+
+  Value *charFwd = builder.CreateLoad(charFwdPtr);
+  Value *charRev = builder.CreateLoad(charRevPtr);
+
+  GlobalVariable *table = types::ByteType::getByteCompTable(block->getModule());
+  charRev = builder.CreateZExt(charRev, builder.getInt64Ty());
+  charRev = builder.CreateInBoundsGEP(table, {builder.getInt64(0), charRev});
+  charRev = builder.CreateLoad(charRev);
+
+  return builder.CreateSelect(rc, charRev, charFwd);
+}
+
 static Value *codegenSeqMatchForSeq(const std::vector<char> &patterns,
                                     BaseFunc *base, types::Type *type,
                                     Value *val, BasicBlock *&block) {
@@ -330,14 +353,16 @@ static Value *codegenSeqMatchForSeq(const std::vector<char> &patterns,
   Value *lenMatch = nullptr;
   BasicBlock *startBlock = block;
   IRBuilder<> builder(block);
+  Value *rc = builder.CreateICmpSLT(len, zeroLLVM(context));
+  Value *lenActual = builder.CreateSelect(rc, builder.CreateNeg(len), len);
 
   // check lengths:
   if (hasStar) {
     Value *minLen = ConstantInt::get(seqIntLLVM(context), patterns.size() - 1);
-    lenMatch = builder.CreateICmpSGE(len, minLen);
+    lenMatch = builder.CreateICmpSGE(lenActual, minLen);
   } else {
     Value *expectedLen = ConstantInt::get(seqIntLLVM(context), patterns.size());
-    lenMatch = builder.CreateICmpEQ(len, expectedLen);
+    lenMatch = builder.CreateICmpEQ(lenActual, expectedLen);
   }
 
   block = BasicBlock::Create(
@@ -353,7 +378,7 @@ static Value *codegenSeqMatchForSeq(const std::vector<char> &patterns,
       if (patterns[i] == '_')
         continue;
       Value *idx = ConstantInt::get(seqIntLLVM(context), i);
-      Value *sub = builder.CreateLoad(builder.CreateGEP(ptr, idx));
+      Value *sub = indexIntoSeq(ptr, lenActual, rc, idx, block);
       Value *c = ConstantInt::get(IntegerType::getInt8Ty(context),
                                   (uint64_t)patterns[i]);
       Value *subRes = builder.CreateICmpEQ(sub, c);
@@ -366,11 +391,11 @@ static Value *codegenSeqMatchForSeq(const std::vector<char> &patterns,
       if (patterns[i] == '_')
         continue;
       Value *idx = ConstantInt::get(seqIntLLVM(context), i);
-      idx = builder.CreateAdd(idx, len);
+      idx = builder.CreateAdd(idx, lenActual);
       idx = builder.CreateSub(
           idx, ConstantInt::get(seqIntLLVM(context), patterns.size()));
 
-      Value *sub = builder.CreateLoad(builder.CreateGEP(ptr, idx));
+      Value *sub = indexIntoSeq(ptr, lenActual, rc, idx, block);
       Value *c = ConstantInt::get(IntegerType::getInt8Ty(context),
                                   (uint64_t)patterns[i]);
       Value *subRes = builder.CreateICmpEQ(sub, c);
@@ -383,7 +408,7 @@ static Value *codegenSeqMatchForSeq(const std::vector<char> &patterns,
       if (patterns[i] == '_')
         continue;
       Value *idx = ConstantInt::get(seqIntLLVM(context), i);
-      Value *sub = builder.CreateLoad(builder.CreateGEP(ptr, idx));
+      Value *sub = indexIntoSeq(ptr, lenActual, rc, idx, block);
       Value *c = ConstantInt::get(IntegerType::getInt8Ty(context),
                                   (uint64_t)patterns[i]);
       Value *subRes = builder.CreateICmpEQ(sub, c);
@@ -413,9 +438,13 @@ static Value *codegenSeqMatchForSeq(const std::vector<char> &patterns,
 static Value *codegenSeqMatchForKmer(const std::vector<char> &patterns,
                                      BaseFunc *base, types::KMer *type,
                                      Value *val, BasicBlock *&block) {
+  static const unsigned BRUTE_MATCH_K_CUTOFF = 256;
+  static const unsigned BRUTE_MATCH_P_CUTOFF = 100;
+
   LLVMContext &context = block->getContext();
   Value *fail = ConstantInt::get(IntegerType::getInt1Ty(context), 0);
   Value *succ = ConstantInt::get(IntegerType::getInt1Ty(context), 1);
+  Value *falseBool = ConstantInt::get(types::Bool->getLLVMType(context), 0);
 
   if (patterns.empty())
     return fail; // k-mer length must be at least 1
@@ -451,11 +480,59 @@ static Value *codegenSeqMatchForKmer(const std::vector<char> &patterns,
   }
 
   Type *llvmType = type->getLLVMType(context);
+
+  // if k is small and pattern is small, brute force matching is fastest:
+  if (hasStar && k <= BRUTE_MATCH_K_CUTOFF &&
+      patterns.size() <= BRUTE_MATCH_P_CUTOFF) {
+    types::KMer *k1Type = types::KMer::get(1);
+
+    BasicBlock *succBlock = block;
+    BasicBlock *failBlock = BasicBlock::Create(context, "", block->getParent());
+
+    IRBuilder<> builder(succBlock);
+    bool backIndex = false;
+
+    for (unsigned i = 0; i < patterns.size(); i++) {
+      const char c = patterns[i];
+      if (c == '_')
+        continue;
+      if (c == '\0') {
+        assert(!backIndex);
+        backIndex = true;
+        continue;
+      }
+      unsigned idx = backIndex ? (k + i - patterns.size()) : i;
+      Value *idxVal = ConstantInt::get(seqIntLLVM(context), idx);
+      Value *base = type->callMagic("__getitem__", {types::Int}, val, {idxVal},
+                                    succBlock, nullptr);
+      Value *expected =
+          k1Type->callMagic("__init__", {types::Byte}, nullptr,
+                            {builder.getInt8(c)}, succBlock, nullptr);
+      Value *match = builder.CreateICmpEQ(base, expected);
+      succBlock = BasicBlock::Create(context, "", block->getParent());
+      builder.CreateCondBr(match, succBlock, failBlock);
+      builder.SetInsertPoint(succBlock);
+    }
+
+    block = BasicBlock::Create(context, "", block->getParent());
+    builder.CreateBr(block);
+
+    builder.SetInsertPoint(failBlock);
+    builder.CreateBr(block);
+
+    builder.SetInsertPoint(block);
+    PHINode *result = builder.CreatePHI(IntegerType::getInt1Ty(context), 2);
+    result->addIncoming(succ, succBlock);
+    result->addIncoming(fail, failBlock);
+    return result;
+  }
+
   if (!hasStar) {
     SeqExpr s(std::string(patterns.begin(), patterns.end()));
     Value *expectedSeq = s.codegen(base, block);
-    Value *expectedKmer = type->callMagic("__init__", {types::Seq}, nullptr,
-                                          {expectedSeq}, block, nullptr);
+    Value *expectedKmer =
+        type->callMagic("__init__", {types::Seq, types::Bool}, nullptr,
+                        {expectedSeq, falseBool}, block, nullptr);
     IRBuilder<> builder(block);
 
     if (!hasWildcard) {
@@ -488,8 +565,9 @@ static Value *codegenSeqMatchForKmer(const std::vector<char> &patterns,
       SeqExpr sLeft(leftString);
       Value *expectedSeqLeft = sLeft.codegen(base, block);
       typeLeft = types::KMer::get(leftString.size());
-      expectedKmerLeft = typeLeft->callMagic("__init__", {types::Seq}, nullptr,
-                                             {expectedSeqLeft}, block, nullptr);
+      expectedKmerLeft =
+          typeLeft->callMagic("__init__", {types::Seq, types::Bool}, nullptr,
+                              {expectedSeqLeft, falseBool}, block, nullptr);
     }
 
     if (!rightString.empty()) {

compiler/types/funct.cpp

@@ -168,7 +168,7 @@ void types::GenType::resume(Value *self, BasicBlock *block, BasicBlock *normal,
     builder.CreateCall(resFn, self);
 }
 
-Value *types::GenType::promise(Value *self, BasicBlock *block) {
+Value *types::GenType::promise(Value *self, BasicBlock *block, bool returnPtr) {
   if (outType->is(types::Void))
     return nullptr;
 
@@ -187,7 +187,15 @@ Value *types::GenType::promise(Value *self, BasicBlock *block) {
   Value *ptr = builder.CreateCall(promFn, {self, aln, from});
   ptr = builder.CreateBitCast(
       ptr, PointerType::get(outType->getLLVMType(context), 0));
-  return builder.CreateLoad(ptr);
+  return returnPtr ? ptr : builder.CreateLoad(ptr);
+}
+
+void types::GenType::send(Value *self, Value *val, BasicBlock *block) {
+  Value *promisePtr = promise(self, block, /*returnPtr=*/true);
+  if (!promisePtr)
+    throw exc::SeqException("cannot send value to void generator");
+  IRBuilder<> builder(block);
+  builder.CreateStore(val, promisePtr);
 }
 
 void types::GenType::destroy(Value *self, BasicBlock *block) {
@@ -269,6 +277,36 @@ void types::GenType::initOps() {
           }),
       true);
 
+  addMethod("send",
+            new BaseFuncLite(
+                {this, outType}, Void,
+                [this](Module *module) {
+                  const std::string name = "seq." + getName() + ".send";
+                  Function *func = module->getFunction(name);
+
+                  if (!func) {
+                    LLVMContext &context = module->getContext();
+                    func = cast<Function>(module->getOrInsertFunction(
+                        name, Void->getLLVMType(context), getLLVMType(context),
+                        outType->getLLVMType(context)));
+                    func->setLinkage(GlobalValue::PrivateLinkage);
+                    func->setDoesNotThrow();
+                    func->addFnAttr(Attribute::AlwaysInline);
+
+                    auto iter = func->arg_begin();
+                    Value *self = iter++;
+                    Value *val = iter;
+                    BasicBlock *entry =
+                        BasicBlock::Create(context, "entry", func);
+                    send(self, val, entry);
+                    IRBuilder<> builder(entry);
+                    builder.CreateRetVoid();
+                  }
+
+                  return func;
+                }),
+            true);
+
   addMethod(
       "destroy",
       new BaseFuncLite(