Skip to content

Commit 82ab7a1

Browse files
Oceania2018Oceania2018
authored
Merge pull request #1361 from ywang1110/master
feat(entity): context-aware loaders + Membase graph-backed NER + per-request auth token
2 parents c5986d4 + 7ad9ef4 commit 82ab7a1

9 files changed

Lines changed: 352 additions & 8 deletions

File tree

‎src/Infrastructure/BotSharp.Abstraction/Entity/IEntityDataLoader.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
using BotSharp.Abstraction.Entity.Models;
2+
13
namespace BotSharp.Abstraction.Entity;
24

35
public interface IEntityDataLoader
@@ -15,4 +17,18 @@ public interface IEntityDataLoader
1517
/// </summary>
1618
/// <returns></returns>
1719
Task<Dictionary<string, (string DataSource, string CanonicalForm)>> LoadSynonymMappingAsync();
20+
21+
/// <summary>
22+
/// Context-aware vocabulary load. Default implementation delegates to the
23+
/// parameterless version for loaders that don't need runtime parameters.
24+
/// </summary>
25+
Task<Dictionary<string, HashSet<string>>> LoadVocabularyAsync(EntityDataLoadContext ctx)
26+
=> LoadVocabularyAsync();
27+
28+
/// <summary>
29+
/// Context-aware synonym load. Default implementation delegates to the
30+
/// parameterless version for loaders that don't need runtime parameters.
31+
/// </summary>
32+
Task<Dictionary<string, (string DataSource, string CanonicalForm)>> LoadSynonymMappingAsync(EntityDataLoadContext ctx)
33+
=> LoadSynonymMappingAsync();
1834
}

‎src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityAnalysisOptions.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@ public class EntityAnalysisOptions
99
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
1010
public IEnumerable<string>? DataProviders { get; set; }
1111

12+
/// <summary>
13+
/// Free-form parameters forwarded to <see cref="IEntityDataLoader"/> implementations.
14+
/// Each loader documents the keys it recognizes (e.g. "graphId" for graph-backed loaders).
15+
/// </summary>
16+
[JsonPropertyName("loader_parameters")]
17+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
18+
public IDictionary<string, string>? LoaderParameters { get; set; }
19+
1220
/// <summary>
1321
/// Maximum n-gram size
1422
/// </summary>

‎src/Infrastructure/BotSharp.Abstraction/Entity/Models/EntityDataLoadContext.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
namespace BotSharp.Abstraction.Entity.Models;
2+
3+
/// <summary>
4+
/// Loader-facing context carrying free-form parameters from the caller
5+
/// (e.g. via <see cref="EntityAnalysisOptions.LoaderParameters"/>).
6+
/// Each <see cref="IEntityDataLoader"/> implementation defines which keys it
7+
/// recognizes (document them on the concrete loader's XML doc).
8+
/// </summary>
9+
public class EntityDataLoadContext
10+
{
11+
/// <summary>
12+
/// Case-insensitive key/value bag (e.g. "graphId", "tenantId").
13+
/// </summary>
14+
public IDictionary<string, string> Parameters { get; init; }
15+
= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
16+
}

‎src/Plugins/BotSharp.Plugin.FuzzySharp/FuzzySharpPlugin.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
2222
services.AddScoped<IResultProcessor, ResultProcessor>();
2323
services.AddScoped<IEntityAnalyzer, FuzzySharpEntityAnalyzer>();
2424
services.AddScoped<IEntityDataLoader, CsvNERDataLoader>();
25+
services.AddScoped<MembaseNERDataLoader>();
26+
services.AddScoped<IEntityDataLoader>(sp => sp.GetRequiredService<MembaseNERDataLoader>());
2527

2628
services.AddScoped<ITokenMatcher, ExactMatcher>();
2729
services.AddScoped<ITokenMatcher, SynonymMatcher>();

‎src/Plugins/BotSharp.Plugin.FuzzySharp/Services/DataLoaders/MembaseNERDataLoader.cs‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
using BotSharp.Abstraction.Graph;
2+
using BotSharp.Abstraction.Graph.Options;
3+
using BotSharp.Abstraction.Infrastructures;
4+
using Microsoft.Extensions.Logging;
5+
using System.Text.RegularExpressions;
6+
7+
namespace BotSharp.Plugin.FuzzySharp.Services.DataLoaders;
8+
9+
/// <summary>
10+
/// Loads NER vocabulary / synonyms from a graph database (e.g. Membase) per-call.
11+
/// Required context parameter:
12+
/// - "graphId": target graph identifier (non-empty string).
13+
/// Vocabulary schemas are configured per-tenant under
14+
/// <c>FuzzySharp:Membase:Tenants:&lt;alias&gt;:Schema</c> (committed in appsettings),
15+
/// with the tenant's environment-specific <c>GraphId</c> supplied via user-secrets
16+
/// or appsettings.{Environment}.json. The loader resolves the incoming graphId to a
17+
/// tenant via the merged configuration. Each label yields
18+
/// <c>MATCH (n:Label) RETURN n.Property AS text</c>. Synonyms still read the flat
19+
/// (:Synonym {table, column, term, canonical_form}) schema.
20+
/// Exposes InvalidateCacheAsync(graphId) so write paths can force a refresh.
21+
/// </summary>
22+
public class MembaseNERDataLoader : IEntityDataLoader
23+
{
24+
private const string GraphIdKey = "graphId";
25+
private const string CacheKeyPrefix = "fuzzysharp:ner";
26+
private const int CacheMinutes = 60;
27+
private const string GraphDbProvider = "membase";
28+
29+
private const string SynonymCypher =
30+
"MATCH (n:Synonym) RETURN n.table AS table, n.column AS column, n.term AS term, n.canonical_form AS canonical_form";
31+
32+
private static readonly Regex IdentifierRegex = new("^[A-Za-z_][A-Za-z0-9_]*$", RegexOptions.Compiled);
33+
34+
private readonly ILogger<MembaseNERDataLoader> _logger;
35+
private readonly IEnumerable<IGraphDb> _graphDbs;
36+
private readonly ICacheService _cache;
37+
private readonly FuzzySharpSettings _settings;
38+
39+
public MembaseNERDataLoader(
40+
ILogger<MembaseNERDataLoader> logger,
41+
IEnumerable<IGraphDb> graphDbs,
42+
ICacheService cache,
43+
FuzzySharpSettings settings)
44+
{
45+
_logger = logger;
46+
_graphDbs = graphDbs;
47+
_cache = cache;
48+
_settings = settings;
49+
}
50+
51+
public string Provider => "fuzzy-sharp-membase";
52+
53+
private static string VocabKey(string graphId) => $"{CacheKeyPrefix}:vocab:{graphId}";
54+
private static string SynonymKey(string graphId) => $"{CacheKeyPrefix}:synonym:{graphId}";
55+
56+
// The parameterless overloads don't make sense for a graph-backed loader.
57+
// Caller must supply a graphId via EntityDataLoadContext.
58+
public Task<Dictionary<string, HashSet<string>>> LoadVocabularyAsync()
59+
=> Task.FromResult(new Dictionary<string, HashSet<string>>());
60+
61+
public Task<Dictionary<string, (string DataSource, string CanonicalForm)>> LoadSynonymMappingAsync()
62+
=> Task.FromResult(new Dictionary<string, (string DataSource, string CanonicalForm)>());
63+
64+
public Task<Dictionary<string, HashSet<string>>> LoadVocabularyAsync(EntityDataLoadContext ctx)
65+
{
66+
if (!TryGetGraphId(ctx, out var graphId))
67+
{
68+
return Task.FromResult(new Dictionary<string, HashSet<string>>());
69+
}
70+
return LoadVocabularyByGraphIdAsync(graphId);
71+
}
72+
73+
public Task<Dictionary<string, (string DataSource, string CanonicalForm)>> LoadSynonymMappingAsync(EntityDataLoadContext ctx)
74+
{
75+
if (!TryGetGraphId(ctx, out var graphId))
76+
{
77+
return Task.FromResult(new Dictionary<string, (string DataSource, string CanonicalForm)>());
78+
}
79+
return LoadSynonymMappingByGraphIdAsync(graphId);
80+
}
81+
82+
private async Task<Dictionary<string, HashSet<string>>> LoadVocabularyByGraphIdAsync(string graphId)
83+
{
84+
var key = VocabKey(graphId);
85+
var cached = await _cache.GetAsync<Dictionary<string, HashSet<string>>>(key);
86+
if (cached != null) return cached;
87+
88+
var result = new Dictionary<string, HashSet<string>>();
89+
90+
var sources = _settings.Membase?.VocabularySources;
91+
if (sources == null || !sources.TryGetValue(graphId, out var labelMap) || labelMap == null || labelMap.Count == 0)
92+
{
93+
_logger.LogWarning($"Skip {Provider}: no vocabulary sources configured for graphId='{graphId}' under FuzzySharp:Membase:VocabularySources.");
94+
return result;
95+
}
96+
97+
var graphDb = ResolveGraphDb();
98+
if (graphDb == null) return result;
99+
100+
foreach (var (label, fields) in labelMap)
101+
{
102+
if (fields == null || fields.Length == 0) continue;
103+
104+
if (!IdentifierRegex.IsMatch(label))
105+
{
106+
_logger.LogWarning($"Skip vocabulary label '{label}' in {Provider}: invalid identifier.");
107+
continue;
108+
}
109+
110+
// Build aliased projections: n.prop0 AS f0, n.prop1 AS f1, ...
111+
// Carry SqlSource alongside so we can key the result dict by SQL "table.column".
112+
var validFields = new List<(string Alias, string GraphProperty, string SqlSource)>(fields.Length);
113+
for (var i = 0; i < fields.Length; i++)
114+
{
115+
var graphProperty = fields[i].GraphProperty;
116+
var sqlSource = fields[i].SqlSource;
117+
if (string.IsNullOrWhiteSpace(graphProperty) || !IdentifierRegex.IsMatch(graphProperty))
118+
{
119+
_logger.LogWarning($"Skip vocabulary field '{label}.{graphProperty}' in {Provider}: invalid identifier.");
120+
continue;
121+
}
122+
if (string.IsNullOrWhiteSpace(sqlSource))
123+
{
124+
_logger.LogWarning($"Skip vocabulary field '{label}.{graphProperty}' in {Provider}: empty SqlSource.");
125+
continue;
126+
}
127+
validFields.Add(($"f{i}", graphProperty, sqlSource));
128+
}
129+
if (validFields.Count == 0) continue;
130+
131+
var projection = string.Join(", ", validFields.Select(f => $"n.{f.GraphProperty} AS {f.Alias}"));
132+
var cypher = $"MATCH (n:{label}) RETURN {projection}";
133+
134+
try
135+
{
136+
var queryResult = await graphDb.ExecuteQueryAsync(cypher, new GraphQueryExecuteOptions
137+
{
138+
GraphId = graphId
139+
});
140+
141+
foreach (var (alias, _, sqlSource) in validFields)
142+
{
143+
if (!result.TryGetValue(sqlSource, out var set))
144+
{
145+
set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
146+
result[sqlSource] = set;
147+
}
148+
149+
foreach (var row in queryResult?.Values ?? [])
150+
{
151+
var text = row.TryGetValue(alias, out var tx) ? tx?.ToString() : null;
152+
if (string.IsNullOrWhiteSpace(text)) continue;
153+
set.Add(text!.Trim());
154+
}
155+
}
156+
}
157+
catch (Exception ex)
158+
{
159+
_logger.LogError(ex, $"Error loading vocabulary label '{label}' from {Provider} graphId={graphId}");
160+
}
161+
}
162+
163+
_logger.LogInformation($"Loaded vocabulary from {Provider} graphId={graphId}: {result.Sum(x => x.Value.Count)} terms across {result.Count} sources");
164+
await _cache.SetAsync(key, result, TimeSpan.FromMinutes(CacheMinutes));
165+
166+
return result;
167+
}
168+
169+
private async Task<Dictionary<string, (string DataSource, string CanonicalForm)>> LoadSynonymMappingByGraphIdAsync(string graphId)
170+
{
171+
var key = SynonymKey(graphId);
172+
var cached = await _cache.GetAsync<Dictionary<string, (string DataSource, string CanonicalForm)>>(key);
173+
if (cached != null) return cached;
174+
175+
var result = new Dictionary<string, (string DataSource, string CanonicalForm)>();
176+
var graphDb = ResolveGraphDb();
177+
if (graphDb == null) return result;
178+
179+
try
180+
{
181+
var queryResult = await graphDb.ExecuteQueryAsync(SynonymCypher, new GraphQueryExecuteOptions
182+
{
183+
GraphId = graphId
184+
});
185+
186+
foreach (var row in queryResult?.Values ?? [])
187+
{
188+
var term = row.TryGetValue("term", out var t) ? t?.ToString() : null;
189+
var table = row.TryGetValue("table", out var tb) ? tb?.ToString() : null;
190+
var column = row.TryGetValue("column", out var co) ? co?.ToString() : null;
191+
var canonical = row.TryGetValue("canonical_form", out var c) ? c?.ToString() : null;
192+
if (string.IsNullOrWhiteSpace(term) || string.IsNullOrWhiteSpace(table) || string.IsNullOrWhiteSpace(column) || string.IsNullOrWhiteSpace(canonical)) continue;
193+
194+
var dbPath = $"{table!.Trim()}.{column!.Trim()}";
195+
result[term!.Trim().ToLowerInvariant()] = (dbPath, canonical!);
196+
}
197+
198+
_logger.LogInformation($"Loaded synonym mapping from {Provider} graphId={graphId}: {result.Count} terms");
199+
await _cache.SetAsync(key, result, TimeSpan.FromMinutes(CacheMinutes));
200+
}
201+
catch (Exception ex)
202+
{
203+
_logger.LogError(ex, $"Error loading synonym mapping from {Provider} graphId={graphId}");
204+
}
205+
206+
return result;
207+
}
208+
209+
public async Task InvalidateCacheAsync(string graphId)
210+
{
211+
if (string.IsNullOrWhiteSpace(graphId)) return;
212+
await _cache.RemoveAsync(VocabKey(graphId));
213+
await _cache.RemoveAsync(SynonymKey(graphId));
214+
}
215+
216+
private IGraphDb? ResolveGraphDb()
217+
{
218+
var graphDb = _graphDbs.FirstOrDefault(x => string.Equals(x.Provider, GraphDbProvider, StringComparison.OrdinalIgnoreCase));
219+
if (graphDb == null)
220+
{
221+
_logger.LogWarning($"No IGraphDb registered with provider '{GraphDbProvider}'. Skip {Provider}.");
222+
}
223+
return graphDb;
224+
}
225+
226+
private bool TryGetGraphId(EntityDataLoadContext ctx, out string graphId)
227+
{
228+
if (ctx.Parameters.TryGetValue(GraphIdKey, out var value) && !string.IsNullOrWhiteSpace(value))
229+
{
230+
graphId = value;
231+
return true;
232+
}
233+
graphId = string.Empty;
234+
_logger.LogWarning($"Skip {Provider}: '{GraphIdKey}' not provided in EntityDataLoadContext.");
235+
return false;
236+
}
237+
}

‎src/Plugins/BotSharp.Plugin.FuzzySharp/Services/FuzzySharpEntityAnalyzer.cs‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -66,11 +66,14 @@ private async Task<TokenAnalysisResponse> AnalyzeTextAsync(string text, EntityAn
6666
// Tokenize the text
6767
var tokens = TokenHelper.Tokenize(text);
6868

69+
// Build loader context once and reuse for both calls
70+
var loaderCtx = BuildLoaderContext(options);
71+
6972
// Load vocabulary
70-
var vocabulary = await LoadAllVocabularyAsync(options?.DataProviders);
73+
var vocabulary = await LoadAllVocabularyAsync(options?.DataProviders, loaderCtx);
7174

7275
// Load synonym mapping
73-
var synonymMapping = await LoadAllSynonymMappingAsync(options?.DataProviders);
76+
var synonymMapping = await LoadAllSynonymMappingAsync(options?.DataProviders, loaderCtx);
7477

7578
// Analyze text
7679
var flaggedItems = AnalyzeTokens(tokens, vocabulary, synonymMapping, options);
@@ -99,10 +102,11 @@ private async Task<TokenAnalysisResponse> AnalyzeTextAsync(string text, EntityAn
99102
}
100103
}
101104

102-
public async Task<Dictionary<string, HashSet<string>>> LoadAllVocabularyAsync(IEnumerable<string>? dataProviders = null)
105+
public async Task<Dictionary<string, HashSet<string>>> LoadAllVocabularyAsync(IEnumerable<string>? dataProviders = null, EntityDataLoadContext? ctx = null)
103106
{
107+
ctx ??= new EntityDataLoadContext();
104108
var dataLoaders = _tokenDataLoaders.Where(x => dataProviders == null || dataProviders.Contains(x.Provider));
105-
var results = await Task.WhenAll(dataLoaders.Select(c => c.LoadVocabularyAsync()));
109+
var results = await Task.WhenAll(dataLoaders.Select(c => c.LoadVocabularyAsync(ctx)));
106110
var merged = new Dictionary<string, HashSet<string>>();
107111

108112
foreach (var dict in results)
@@ -123,10 +127,11 @@ public async Task<Dictionary<string, HashSet<string>>> LoadAllVocabularyAsync(IE
123127
return merged;
124128
}
125129

126-
public async Task<Dictionary<string, (string DbPath, string CanonicalForm)>> LoadAllSynonymMappingAsync(IEnumerable<string>? dataProviders = null)
130+
public async Task<Dictionary<string, (string DbPath, string CanonicalForm)>> LoadAllSynonymMappingAsync(IEnumerable<string>? dataProviders = null, EntityDataLoadContext? ctx = null)
127131
{
132+
ctx ??= new EntityDataLoadContext();
128133
var dataLoaders = _tokenDataLoaders.Where(x => dataProviders == null || dataProviders.Contains(x.Provider));
129-
var results = await Task.WhenAll(dataLoaders.Select(c => c.LoadSynonymMappingAsync()));
134+
var results = await Task.WhenAll(dataLoaders.Select(c => c.LoadSynonymMappingAsync(ctx)));
130135
var merged = new Dictionary<string, (string DbPath, string CanonicalForm)>();
131136

132137
foreach (var dict in results)
@@ -140,6 +145,22 @@ public async Task<Dictionary<string, HashSet<string>>> LoadAllVocabularyAsync(IE
140145
return merged;
141146
}
142147

148+
private static EntityDataLoadContext BuildLoaderContext(EntityAnalysisOptions? options)
149+
{
150+
var ctx = new EntityDataLoadContext();
151+
if (options?.LoaderParameters is { } src)
152+
{
153+
foreach (var kvp in src)
154+
{
155+
if (!string.IsNullOrEmpty(kvp.Key))
156+
{
157+
ctx.Parameters[kvp.Key] = kvp.Value;
158+
}
159+
}
160+
}
161+
return ctx;
162+
}
163+
143164
/// <summary>
144165
/// Analyze tokens for typos and entities
145166
/// </summary>

0 commit comments

Comments
 (0)