Add ignored regression test for optional full-outer-join scalar access kind

src/core/src/program/compiler/constants.rs

@@ -91,10 +91,16 @@ impl CompileConst for Value {
       Value::Table(x) => x.borrow().compile_const(ctx)?,
       #[cfg(feature = "record")]
       Value::Record(x) => x.borrow().compile_const(ctx)?,
-      #[cfg(feature = "set")]
-      Value::Set(x) => x.borrow().compile_const(ctx)?,
-      x => todo!("CompileConst not implemented for {:?}", x),
-    };
+      #[cfg(feature = "set")]
+      Value::Set(x) => x.borrow().compile_const(ctx)?,
+      Value::Typed(value, kind) => match value.as_ref() {
+        Value::Empty => ctx.compile_const(&[], kind.clone())?,
+        _ => value.compile_const(ctx)?,
+      },
+      Value::EmptyKind(k) => ctx.compile_const(&[], k.clone())?,
+      Value::Empty => ctx.compile_const(&[], ValueKind::Empty)?,
+      x => todo!("CompileConst not implemented for {:?}", x),
+    };
     Ok(reg)
   }
 }
@@ -920,9 +926,14 @@ impl ConstElem for Value {
 
     // Then write the payload
     match self {
-      Value::Empty => { 
-        // no payload for Empty 
-      },
+      Value::Empty | Value::EmptyKind(_) => { 
+        // no payload for Empty 
+      },
+      Value::Typed(value, _) => {
+        if !matches!(value.as_ref(), Value::Empty) {
+          unimplemented!("write_le for non-empty typed value is not implemented");
+        }
+      }
       #[cfg(feature = "bool")]
       Value::Bool(x) => x.borrow().write_le(out),
       #[cfg(feature = "string")]
@@ -973,7 +984,8 @@ impl ConstElem for Value {
 
     // 3. dispatch based on ValueKind
     match kind {
-      ValueKind::Empty => Value::Empty,
+      ValueKind::Empty => Value::Empty,
+      ValueKind::Option(inner) => Value::EmptyKind(ValueKind::Option(inner)),
       #[cfg(feature = "bool")]
       ValueKind::Bool => Value::Bool(Ref::new(<bool as ConstElem>::from_le(payload))),
       #[cfg(feature = "string")]

src/core/src/value.rs

@@ -44,8 +44,9 @@ macro_rules! impl_as_type {
           Value::F32(v) => Ok(Ref::new((*v.borrow()) as $target_type)),
           #[cfg(feature = "f64")]
           Value::F64(v) => Ok(Ref::new((*v.borrow()) as $target_type)),
-          Value::Id(v) => Ok(Ref::new(*v as $target_type)),
-          Value::MutableReference(val) => val.borrow().[<as_ $target_type>](),
+          Value::Id(v) => Ok(Ref::new(*v as $target_type)),
+          Value::Typed(value, _) => value.[<as_ $target_type>](),
+          Value::MutableReference(val) => val.borrow().[<as_ $target_type>](),
           _ => Err(
             MechError::new(
               CannotConvertToTypeError { target_type: stringify!($target_type) },
@@ -622,11 +623,13 @@ pub enum Value {
   Enum(Ref<MechEnum>),
   Id(u64),
   Index(Ref<usize>),
-  MutableReference(MutableReference),
-  Kind(ValueKind),
-  IndexAll,
-  Empty
-}
+  MutableReference(MutableReference),
+  Typed(Box<Value>, ValueKind),
+  Kind(ValueKind),
+  IndexAll,
+  EmptyKind(ValueKind),
+  Empty
+}
 
 impl Eq for Value {}
 
@@ -728,11 +731,16 @@ impl Hash for Value {
       #[cfg(all(feature = "matrix", feature = "complex"))]
       Value::MatrixC64(x) => x.hash(state),
       Value::Id(x)   => x.hash(state),
-      Value::Kind(x) => x.hash(state),
-      Value::Index(x)=> x.borrow().hash(state),
-      Value::MutableReference(x) => x.borrow().hash(state),
-      Value::Empty => Value::Empty.hash(state),
-      Value::IndexAll => Value::IndexAll.hash(state),
+      Value::Kind(x) => x.hash(state),
+      Value::Typed(v, k) => {
+        v.hash(state);
+        k.hash(state);
+      }
+      Value::Index(x)=> x.borrow().hash(state),
+      Value::MutableReference(x) => x.borrow().hash(state),
+      Value::EmptyKind(k) => k.hash(state),
+      Value::Empty => Value::Empty.hash(state),
+      Value::IndexAll => Value::IndexAll.hash(state),
     }
   }
 }
@@ -745,9 +753,9 @@ impl Value {
 
     for value in matrix.as_vec().iter() {
       match value {
-        Value::Empty => {
-          saw_empty = true;
-        }
+        Value::Empty | Value::EmptyKind(_) => {
+          saw_empty = true;
+        }
         _ => {
           let kind = value.kind();
           let (normalized_kind, normalized_empty) = match kind {
@@ -1001,7 +1009,9 @@ impl Value {
     }
 
     match (self, other) {
-    (Value::Empty, ValueKind::Option(_)) => Some(Value::Empty),
+    (Value::Typed(value, _), target_kind) => value.convert_to(target_kind),
+    (Value::Empty, ValueKind::Option(_)) => Some(Value::Empty),
+    (Value::EmptyKind(_), ValueKind::Option(_)) => Some(Value::Empty),
     (value, ValueKind::Option(inner)) => value.convert_to(inner.as_ref()),
     (value, ValueKind::Matrix(_, target_shape))
       if target_shape.is_empty() && matches!(value.kind(), ValueKind::Matrix(_, _)) =>
@@ -1385,9 +1395,11 @@ impl Value {
       Value::MutableReference(x) => x.borrow().size_of(),
       Value::Id(_) => 8,
       Value::Index(x) => 8,
-      Value::Kind(_) => 0, // Kind is not a value, so it has no size
-      Value::Empty => 0,
-      Value::IndexAll => 0, // IndexAll is a special value, so it has no size
+      Value::Kind(_) => 0, // Kind is not a value, so it has no size
+      Value::Typed(value, _) => value.size_of(),
+      Value::EmptyKind(_) => 0,
+      Value::Empty => 0,
+      Value::IndexAll => 0, // IndexAll is a special value, so it has no size
     }
   }
 
@@ -1476,12 +1488,13 @@ impl Value {
       Value::Tuple(t) => t.borrow().to_html(),
       #[cfg(feature = "enum")]
       Value::Enum(e) => e.borrow().to_html(),
-      Value::Empty => "<span class='mech-empty'>_</span>".to_string(),
-      Value::MutableReference(m) => {
-        let inner = m.borrow();
-        format!("<span class='mech-reference'>{}</span>", inner.to_html())
-      },
-      _ => "???".to_string(),
+      Value::Empty | Value::EmptyKind(_) => "<span class='mech-empty'>_</span>".to_string(),
+      Value::MutableReference(m) => {
+        let inner = m.borrow();
+        format!("<span class='mech-reference'>{}</span>", inner.to_html())
+      },
+      Value::Typed(value, _) => value.to_html(),
+      _ => "???".to_string(),
     }
   }
 
@@ -1609,10 +1622,12 @@ impl Value {
       }
       Value::Id(x) => format!("{}", humanize(x)),
       Value::Index(x) => format!("{}", x.borrow()),
-      Value::Kind(k) => format!("<{}>", k),
-      Value::MutableReference(m) => m.borrow().format_value_inline(),
-      Value::IndexAll => ":".to_string(),
-      Value::Empty => "_".to_string(),
+      Value::Kind(k) => format!("<{}>", k),
+      Value::Typed(value, _) => value.format_value_inline(),
+      Value::MutableReference(m) => m.borrow().format_value_inline(),
+      Value::IndexAll => ":".to_string(),
+      Value::EmptyKind(_) => "_".to_string(),
+      Value::Empty => "_".to_string(),
     }
   }
 
@@ -1718,8 +1733,9 @@ impl Value {
       #[cfg(feature = "tuple")]
       Value::Tuple(x) => vec![1,x.borrow().size()],
       Value::Index(x) => vec![1,1],
-      Value::MutableReference(x) => x.borrow().shape(),
-      Value::Empty => vec![0,0],
+      Value::MutableReference(x) => x.borrow().shape(),
+      Value::Typed(x, _) => x.shape(),
+      Value::Empty | Value::EmptyKind(_) => vec![0,0],
       Value::IndexAll => vec![0,0],
       Value::Kind(_) => vec![0,0],
       Value::Id(x) => vec![0,0],
@@ -1817,8 +1833,10 @@ impl Value {
       Value::Tuple(x) => x.borrow().kind(),
       #[cfg(feature = "enum")]
       Value::Enum(x) => x.borrow().kind(),
-      Value::MutableReference(x) => ValueKind::Reference(Box::new(x.borrow().kind())),
-      Value::Empty => ValueKind::Empty,
+      Value::MutableReference(x) => ValueKind::Reference(Box::new(x.borrow().kind())),
+      Value::Typed(_, kind) => kind.clone(),
+      Value::EmptyKind(k) => k.clone(),
+      Value::Empty => ValueKind::Empty,
       Value::IndexAll => ValueKind::Empty,
       Value::Id(x) => ValueKind::Id,
       Value::Index(x) => ValueKind::Index,
@@ -2438,8 +2456,9 @@ impl PrettyPrint for Value {
       Value::MatrixC64(x) => {return x.pretty_print();},
       Value::MatrixValue(x) => {return x.pretty_print();},
       Value::Index(x)  => {builder.push_record(vec![format!("{}",x.borrow())]);},
-      Value::MutableReference(x) => {return x.borrow().pretty_print();},
-      Value::Empty => builder.push_record(vec!["_"]),
+      Value::MutableReference(x) => {return x.borrow().pretty_print();},
+      Value::Typed(x, _) => { return x.pretty_print(); },
+      Value::Empty | Value::EmptyKind(_) => builder.push_record(vec!["_"]),
       Value::IndexAll => builder.push_record(vec![":"]),
       Value::Id(x) => builder.push_record(vec![format!("{}",humanize(x))]),
       Value::Kind(x) => builder.push_record(vec![format!("<{}>",x)]),

src/interpreter/src/expressions.rs

@@ -1141,13 +1141,14 @@ fn guard_expression_true(guard: &Expression, env: &Environment, p: &Interpreter)
 
 fn value_contains_empty(value: &Value) -> bool {
     match value {
-        Value::Empty => true,
+        Value::Empty | Value::EmptyKind(_) => true,
         #[cfg(feature = "tuple")]
         Value::Tuple(tuple) => tuple
             .borrow()
             .elements
             .iter()
             .any(|value| value_contains_empty(value.as_ref())),
+        Value::Typed(value, _) => value_contains_empty(value),
         Value::MutableReference(reference) => value_contains_empty(&reference.borrow()),
         _ => false,
     }

src/interpreter/src/stdlib/access/matrix.rs

@@ -823,12 +823,17 @@ struct MatrixAccessScalarValueF {
   source: Matrix<Value>,
   ix: Ref<usize>,
   out: Ref<Value>,
+  element_kind: ValueKind,
 }
 
 impl MechFunctionImpl for MatrixAccessScalarValueF {
   fn solve(&self) {
     let ix = *self.ix.borrow();
-    *self.out.borrow_mut() = self.source.index1d(ix);
+    let value = self.source.index1d(ix);
+    *self.out.borrow_mut() = match &self.element_kind {
+      ValueKind::Option(_) => Value::Typed(Box::new(value), self.element_kind.clone()),
+      _ => value,
+    };
   }
   fn out(&self) -> Value { self.out.borrow().clone() }
   fn to_string(&self) -> String { format!("{:#?}", self) }
@@ -861,10 +866,19 @@ impl NativeFunctionCompiler for MatrixAccessScalar {
     let ixes = arguments.clone().split_off(1);
     let mat = arguments[0].clone();
     if let (Value::MatrixValue(source), [Value::Index(ix)]) = (mat.clone(), ixes.as_slice()) {
+      let element_kind = match mat.kind() {
+        ValueKind::Matrix(elem, _) => (*elem).clone(),
+        _ => ValueKind::Any,
+      };
+      let init = match &element_kind {
+        ValueKind::Option(_) => Value::Typed(Box::new(Value::Empty), element_kind.clone()),
+        _ => Value::Empty,
+      };
       return Ok(Box::new(MatrixAccessScalarValueF {
         source,
         ix: ix.clone(),
-        out: Ref::new(Value::Empty),
+        out: Ref::new(init),
+        element_kind,
       }));
     }
     match impl_access_scalar_fxn(mat.clone(), ixes.clone()) {

src/interpreter/src/stdlib/convert/scalar.rs

@@ -396,6 +396,15 @@ where
 }
 
 fn impl_conversion_fxn(source_value: Value, target_kind: Value) -> MResult<Box<dyn MechFunction>>  {
+  if let (Value::Typed(inner, declared_kind), Value::Kind(target_vk)) = (&source_value, &target_kind) {
+    if declared_kind == target_vk {
+      return Ok(Box::new(ConvertSEmpty {
+        out: Ref::new(Value::Typed(inner.clone(), declared_kind.clone())),
+      }));
+    }
+    return impl_conversion_fxn((**inner).clone(), target_kind);
+  }
+
   match (&source_value, &target_kind) {
     #[cfg(all(feature = "matrix", feature = "set"))]
     (source, Value::Kind(ValueKind::Set(target_kind, _))) => {

src/interpreter/src/stdlib/define.rs

@@ -312,6 +312,8 @@ fn impl_var_define_fxn(var: Value, name: Value, mutable: Value, id: u64) -> MRes
       })));
     },
     (Value::Empty, name, mutable, id) => return box_mech_fxn(Ok(Box::new(VariableDefineEmpty { var: Ref::new(Value::Empty), name: name.as_string()?, mutable: mutable.as_bool()?, id } ))),
+    (Value::Typed(value, kind), name, mutable, id) => return box_mech_fxn(Ok(Box::new(VariableDefineEmpty { var: Ref::new(Value::Typed(value.clone(), kind.clone())), name: name.as_string()?, mutable: mutable.as_bool()?, id } ))),
+    (Value::EmptyKind(kind), name, mutable, id) => return box_mech_fxn(Ok(Box::new(VariableDefineEmpty { var: Ref::new(Value::EmptyKind(kind.clone())), name: name.as_string()?, mutable: mutable.as_bool()?, id } ))),
     #[cfg(feature = "matrix")]
     (Value::MatrixValue(sink), name, mutable, id) => return box_mech_fxn(Ok(Box::new(VariableDefineEmpty { var: Ref::new(Value::MatrixValue(sink.clone())), name: name.as_string()?, mutable: mutable.as_bool()?, id } ))),
     #[cfg(feature = "table")]

tests/interpreter.rs

@@ -1111,6 +1111,48 @@ y<u8> := x.hw1[4]? | x => x | * => 0."#,
   Value::U8(Ref::new(0))
 );
 
+#[cfg(all(feature = "table", feature = "u64", feature = "u8"))]
+test_interpreter!(
+  interpret_table_full_outer_join_optional_column_scalar_access_kind_is_u8_option,
+  r#"a := |id<u64> hw1<u8>| 1 10 | 2 20 | 3 30 |
+b := |id<u64> hw2<u8>| 2 200 | 3 255 | 4 42 |
+x := a ⟗ b
+y := x.hw1[4]
+y"#,
+  Value::MutableReference(Ref::new(Value::Typed(
+    Box::new(Value::Empty),
+    ValueKind::Option(Box::new(ValueKind::U8))
+  )))
+);
+
+#[cfg(all(feature = "table", feature = "u64", feature = "u8"))]
+test_interpreter!(
+  interpret_table_full_outer_join_optional_column_scalar_access_kind_is_u8_option_some,
+  r#"a := |id<u64> hw1<u8>| 1 10 | 2 20 | 3 30 |
+b := |id<u64> hw2<u8>| 2 200 | 3 255 | 4 42 |
+x := a ⟗ b
+y := x.hw1[1]
+y"#,
+  Value::MutableReference(Ref::new(Value::Typed(
+    Box::new(Value::U8(Ref::new(10))),
+    ValueKind::Option(Box::new(ValueKind::U8))
+  )))
+);
+
+#[cfg(all(feature = "table", feature = "u64", feature = "bool"))]
+test_interpreter!(
+  interpret_table_full_outer_join_optional_column_scalar_access_kind_is_bool_option,
+  r#"a := |id<u64> hw1<bool>| 1 true | 2 false | 3 true |
+b := |id<u64> hw2<bool>| 2 true | 3 false | 4 true |
+x := a ⟗ b
+y := x.hw1[1]
+z := x.hw1[4]"#,
+  Value::Typed(
+    Box::new(Value::Empty),
+    ValueKind::Option(Box::new(ValueKind::Bool))
+  )
+);
+
 #[cfg(all(feature = "table", feature = "u64"))]
 test_interpreter!(interpret_table_left_semi_join_symbol, r#"A := |id<u64> a<u64>| 1 10 | 2 20 | 3 30 |; B := |id<u64> b<u64>| 2 200 | 3 300 | 4 400 |; J := A ⋉ B; J.a[2]"#, Value::U64(Ref::new(30)));
 #[cfg(all(feature = "table", feature = "u64"))]