Merge pull request #30 from the-dev-tools/feat/add-swagger-openapi-imports

feat(import): add Swagger/OpenAPI import and URL fetching support

Author: ElecTwix

packages/client/src/widgets/import/index.tsx

@@ -150,14 +150,14 @@ const InitialDialog = ({ setModal, successAction }: InitialDialogProps) => {
         `}
       >
         <FiInfo className={tw`mr-1.5 inline-block size-4 align-bottom`} />
-        Import Postman or HAR files
+        Import Postman, HAR, Swagger or OpenAPI files, or paste a URL
       </div>
 
       <TextInputField
         className={tw`mt-4`}
         label='Text value'
         onChange={setText}
-        placeholder='Paste cURL, Raw text or URL...'
+        placeholder='Paste cURL, Swagger/OpenAPI URL, or raw text...'
         value={text}
       />
 

packages/server/internal/api/rimportv2/format_detection.go

@@ -25,6 +25,7 @@ const (
 	FormatJSON
 	FormatCURL
 	FormatPostman
+	FormatOpenAPI
 )
 
 const ReasonValidJSON = "Valid JSON; "
@@ -42,6 +43,8 @@ func (f Format) String() string {
 		return "CURL"
 	case FormatPostman:
 		return "Postman"
+	case FormatOpenAPI:
+		return "OpenAPI"
 	default:
 		return "Unknown"
 	}
@@ -57,19 +60,27 @@ type DetectionResult struct {
 // FormatDetector implements automatic format detection with confidence scoring
 type FormatDetector struct {
 	// Pre-compiled regular expressions for performance
-	harPattern     *regexp.Regexp
-	curlPattern    *regexp.Regexp
-	postmanPattern *regexp.Regexp
-	yamlPattern    *regexp.Regexp
+	harPattern       *regexp.Regexp
+	curlPattern      *regexp.Regexp
+	postmanPattern   *regexp.Regexp
+	yamlPattern      *regexp.Regexp
+	swaggerPattern   *regexp.Regexp
+	openapi3Pattern  *regexp.Regexp
+	yamlSwaggerPat   *regexp.Regexp
+	yamlOpenapi3Pat  *regexp.Regexp
 }
 
 // NewFormatDetector creates a new format detector with compiled patterns
 func NewFormatDetector() *FormatDetector {
 	return &FormatDetector{
-		harPattern:     regexp.MustCompile(`^\s*\{?\s*"?log"?[\s\S]*"?entries"?[\s\S]*\}?\s*$`),
-		curlPattern:    regexp.MustCompile(`(?i)^\s*curl\s+`),
-		postmanPattern: regexp.MustCompile(`(?i)"?info"?\s*:\s*\{[\s\S]*"?name"?[\s\S]*"?schema"?\s*:\s*"https://schema\.getpostman\.com/json/collection/v2\.1\.0/collection\.json"`),
-		yamlPattern:    regexp.MustCompile(`(?i)^\s*flows?\s*:`),
+		harPattern:      regexp.MustCompile(`^\s*\{?\s*"?log"?[\s\S]*"?entries"?[\s\S]*\}?\s*$`),
+		curlPattern:     regexp.MustCompile(`(?i)^\s*curl\s+`),
+		postmanPattern:  regexp.MustCompile(`(?i)"?info"?\s*:\s*\{[\s\S]*"?name"?[\s\S]*"?schema"?\s*:\s*"https://schema\.getpostman\.com/json/collection/v2\.1\.0/collection\.json"`),
+		yamlPattern:     regexp.MustCompile(`(?i)^\s*flows?\s*:`),
+		swaggerPattern:  regexp.MustCompile(`(?i)"swagger"\s*:\s*"2\.\d+"`),
+		openapi3Pattern: regexp.MustCompile(`(?i)"openapi"\s*:\s*"3\.\d+\.\d+"`),
+		yamlSwaggerPat:  regexp.MustCompile(`(?im)^swagger\s*:\s*["']?2\.\d+`),
+		yamlOpenapi3Pat: regexp.MustCompile(`(?im)^openapi\s*:\s*["']?3\.\d+`),
 	}
 }
 
@@ -91,6 +102,7 @@ func (fd *FormatDetector) DetectFormat(data []byte) *DetectionResult {
 	results := []*DetectionResult{
 		fd.detectHAR(trimmed),
 		fd.detectPostman(trimmed),
+		fd.detectOpenAPI(trimmed),
 		fd.detectCURL(trimmed),
 		fd.detectYAML(trimmed),
 		fd.detectJSON(trimmed),
@@ -403,6 +415,91 @@ func (fd *FormatDetector) detectYAML(content string) *DetectionResult {
 	}
 }
 
+// detectOpenAPI detects OpenAPI/Swagger spec format with confidence scoring.
+// Supports both Swagger 2.0 ("swagger": "2.0") and OpenAPI 3.x ("openapi": "3.x.x").
+func (fd *FormatDetector) detectOpenAPI(content string) *DetectionResult {
+	confidence := 0.0
+	reason := ""
+
+	// Check for Swagger 2.0 pattern (JSON)
+	if fd.swaggerPattern.MatchString(content) {
+		confidence += 0.9
+		reason += "Swagger 2.0 spec detected; "
+	}
+
+	// Check for OpenAPI 3.x pattern (JSON)
+	if fd.openapi3Pattern.MatchString(content) {
+		confidence += 0.9
+		reason += "OpenAPI 3.x spec detected; "
+	}
+
+	// Check for YAML-format Swagger 2.0
+	if fd.yamlSwaggerPat.MatchString(content) {
+		confidence += 0.9
+		reason += "Swagger 2.0 YAML spec detected; "
+	}
+
+	// Check for YAML-format OpenAPI 3.x
+	if fd.yamlOpenapi3Pat.MatchString(content) {
+		confidence += 0.9
+		reason += "OpenAPI 3.x YAML spec detected; "
+	}
+
+	// Look for paths field (key indicator of OpenAPI/Swagger)
+	if strings.Contains(content, `"paths"`) || strings.Contains(content, "paths:") {
+		confidence += 0.2
+		reason += "paths field found; "
+	}
+
+	// Look for info field
+	if strings.Contains(content, `"info"`) || strings.Contains(content, "info:") {
+		confidence += 0.1
+		reason += "info field found; "
+	}
+
+	// Validate it's valid JSON or YAML
+	var jsonData map[string]interface{}
+	if err := json.Unmarshal([]byte(content), &jsonData); err == nil {
+		confidence += 0.1
+		reason += ReasonValidJSON
+	} else {
+		// Try YAML
+		var yamlData map[string]interface{}
+		if err := yaml.Unmarshal([]byte(content), &yamlData); err == nil {
+			confidence += 0.1
+			reason += "Valid YAML; "
+		}
+	}
+
+	confidence = max(min(confidence, 1.0), 0)
+
+	return &DetectionResult{
+		Format:     FormatOpenAPI,
+		Confidence: confidence,
+		Reason:     strings.TrimSpace(reason),
+	}
+}
+
+// validateOpenAPI validates OpenAPI/Swagger format specifically
+func (fd *FormatDetector) validateOpenAPI(data []byte) error {
+	// Try JSON first, then YAML
+	var raw map[string]interface{}
+	if err := json.Unmarshal(data, &raw); err != nil {
+		if err := yaml.Unmarshal(data, &raw); err != nil {
+			return fmt.Errorf("invalid OpenAPI spec: not valid JSON or YAML: %w", err)
+		}
+	}
+
+	// Must have either "swagger" or "openapi" field
+	_, hasSwagger := raw["swagger"]
+	_, hasOpenAPI := raw["openapi"]
+	if !hasSwagger && !hasOpenAPI {
+		return fmt.Errorf("missing 'swagger' or 'openapi' version field")
+	}
+
+	return nil
+}
+
 // detectJSON detects generic JSON format with confidence scoring
 func (fd *FormatDetector) detectJSON(content string) *DetectionResult {
 	confidence := 0.0
@@ -486,6 +583,8 @@ func (fd *FormatDetector) ValidateFormat(data []byte, format Format) error {
 		return fd.validateYAML(data)
 	case FormatJSON:
 		return fd.validateJSON(data)
+	case FormatOpenAPI:
+		return fd.validateOpenAPI(data)
 	default:
 		return fmt.Errorf("unknown format: %v", format)
 	}

packages/server/internal/api/rimportv2/service.go

@@ -125,7 +125,7 @@ type ImportConstraints struct {
 func DefaultConstraints() *ImportConstraints {
 	return &ImportConstraints{
 		MaxDataSizeBytes: 50 * 1024 * 1024, // 50MB
-		SupportedFormats: []Format{FormatHAR, FormatYAML, FormatJSON, FormatCURL, FormatPostman},
+		SupportedFormats: []Format{FormatHAR, FormatYAML, FormatJSON, FormatCURL, FormatPostman, FormatOpenAPI},
 		AllowedMimeTypes: []string{
 			"application/json",
 			"application/har",
@@ -218,6 +218,13 @@ func WithLogger(logger *slog.Logger) ServiceOption {
 	}
 }
 
+// WithURLFetcher sets a custom URL fetcher (useful for testing)
+func WithURLFetcher(fetcher URLFetcher) ServiceOption {
+	return func(s *Service) {
+		s.urlFetcher = fetcher
+	}
+}
+
 // WithHTTPService sets the HTTP service for the service (required for HAR import overwrite detection)
 func WithHTTPService(httpService *shttp.HTTPService) ServiceOption {
 	return func(s *Service) {
@@ -231,6 +238,7 @@ type Service struct {
 	importer           Importer
 	validator          Validator
 	translatorRegistry *TranslatorRegistry
+	urlFetcher         URLFetcher
 	logger             *slog.Logger
 	timeout            time.Duration
 }
@@ -244,6 +252,7 @@ func NewService(importer Importer, validator Validator, opts ...ServiceOption) *
 		importer:           importer,
 		validator:          validator,
 		translatorRegistry: NewTranslatorRegistry(nil), // Auto-initialize translator registry without HTTP service (will be overridden if provided)
+		urlFetcher:         NewURLFetcher(),
 		timeout:            30 * time.Minute,           // Default timeout for import processing
 		logger:             slog.Default(),             // Default logger
 	}
@@ -461,7 +470,11 @@ func (s *Service) ImportWithTextData(ctx context.Context, req *ImportRequest) (*
 	return s.Import(ctx, req)
 }
 
-// ImportUnified processes any supported format with automatic detection
+// ImportUnified processes any supported format with automatic detection.
+// It handles three input modes:
+//  1. Binary data (req.Data) - file uploads (Postman JSON, HAR, etc.)
+//  2. Text data (req.TextData) - pasted curl commands, raw JSON/YAML
+//  3. URL text (req.TextData is a URL) - fetches content from URL (e.g., swagger.json URL)
 func (s *Service) ImportUnified(ctx context.Context, req *ImportRequest) (*ImportResults, error) {
 	s.logger.Debug("ImportUnified: Starting", "workspace_id", req.WorkspaceID)
 	// Check if context is already cancelled
@@ -490,6 +503,11 @@ func (s *Service) ImportUnified(ctx context.Context, req *ImportRequest) (*Impor
 		return nil, err // Return the original error for workspace access issues
 	}
 
+	// Resolve input data: handle TextData and URL fetching
+	if err := s.resolveInputData(ctx, req); err != nil {
+		return nil, err
+	}
+
 	s.logger.Debug("ImportUnified: Translating data")
 	// Detect format and translate data
 	translationResult, err := s.importer.ImportAndStoreUnified(ctx, req.Data, req.WorkspaceID)
@@ -689,16 +707,51 @@ func (s *Service) ImportUnifiedWithTextData(ctx context.Context, req *ImportRequ
 		"has_text_data", len(req.TextData) > 0,
 		"has_binary_data", len(req.Data) > 0)
 
-	// Convert text data to bytes if provided
-	if len(req.Data) == 0 && req.TextData != "" {
-		req.Data = []byte(req.TextData)
-		s.logger.Debug("Converted text data to binary",
-			"workspace_id", req.WorkspaceID,
-			"original_length", len(req.TextData),
-			"converted_length", len(req.Data))
+	return s.ImportUnified(ctx, req)
+}
+
+// resolveInputData ensures req.Data is populated for format detection.
+// It handles three cases:
+//  1. Data already present (file upload) - no action needed
+//  2. TextData is a URL - fetch content from the URL
+//  3. TextData is raw text (curl, JSON, YAML) - convert to bytes
+func (s *Service) resolveInputData(ctx context.Context, req *ImportRequest) error {
+	if len(req.Data) > 0 {
+		return nil // Already have binary data from file upload
 	}
 
-	return s.ImportUnified(ctx, req)
+	if req.TextData == "" {
+		return nil // No text data either (validation should have caught this)
+	}
+
+	// Check if TextData is a URL that we should fetch content from
+	if IsURL(req.TextData) {
+		s.logger.Debug("TextData is a URL, fetching content",
+			"url", req.TextData,
+			"workspace_id", req.WorkspaceID)
+
+		data, err := s.urlFetcher.Fetch(ctx, req.TextData)
+		if err != nil {
+			return fmt.Errorf("failed to fetch URL %q: %w", req.TextData, err)
+		}
+
+		req.Data = data
+		if req.Name == "" {
+			req.Name = req.TextData
+		}
+		s.logger.Debug("URL content fetched successfully",
+			"url", req.TextData,
+			"data_size", len(data))
+		return nil
+	}
+
+	// TextData is raw text (curl command, JSON, YAML, etc.)
+	req.Data = []byte(req.TextData)
+	s.logger.Debug("Converted text data to binary",
+		"workspace_id", req.WorkspaceID,
+		"data_size", len(req.Data))
+
+	return nil
 }
 
 // DetectFormat performs format detection on the provided data

packages/server/internal/api/rimportv2/translators.go

@@ -18,6 +18,7 @@ import (
 	"github.com/the-dev-tools/dev-tools/packages/server/pkg/service/shttp"
 	"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/harv2"
 	"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/tcurlv2"
+	"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/topenapiv2"
 	"github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/tpostmanv2"
 	yamlflowsimplev2 "github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/yamlflowsimplev2"
 )
@@ -90,6 +91,7 @@ func NewTranslatorRegistry(httpService *shttp.HTTPService) *TranslatorRegistry {
 	registry.RegisterTranslator(NewYAMLTranslator())
 	registry.RegisterTranslator(NewCURLTranslator())
 	registry.RegisterTranslator(NewPostmanTranslator())
+	registry.RegisterTranslator(NewOpenAPITranslator())
 	registry.RegisterTranslator(NewJSONTranslator())
 
 	return registry
@@ -407,6 +409,56 @@ func (t *PostmanTranslator) Translate(ctx context.Context, data []byte, workspac
 	return result, nil
 }
 
+// OpenAPITranslator implements Translator for OpenAPI/Swagger spec format
+type OpenAPITranslator struct {
+	detector *FormatDetector
+}
+
+// NewOpenAPITranslator creates a new OpenAPI translator
+func NewOpenAPITranslator() *OpenAPITranslator {
+	return &OpenAPITranslator{
+		detector: NewFormatDetector(),
+	}
+}
+
+func (t *OpenAPITranslator) GetFormat() Format {
+	return FormatOpenAPI
+}
+
+func (t *OpenAPITranslator) Validate(data []byte) error {
+	return t.detector.ValidateFormat(data, FormatOpenAPI)
+}
+
+func (t *OpenAPITranslator) Translate(ctx context.Context, data []byte, workspaceID idwrap.IDWrap) (*TranslationResult, error) {
+	opts := topenapiv2.ConvertOptions{
+		WorkspaceID: workspaceID,
+	}
+
+	resolved, err := topenapiv2.ConvertOpenAPI(data, opts)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert OpenAPI spec: %w", err)
+	}
+
+	result := &TranslationResult{
+		HTTPRequests: resolved.HTTPRequests,
+		Files:        resolved.Files,
+		Headers:      resolved.Headers,
+		SearchParams: resolved.SearchParams,
+		BodyRaw:      resolved.BodyRaw,
+		Asserts:      resolved.Asserts,
+		Flows:        []mflow.Flow{resolved.Flow},
+		Nodes:        resolved.Nodes,
+		RequestNodes: resolved.RequestNodes,
+		Edges:        resolved.Edges,
+		ProcessedAt:  time.Now().UnixMilli(),
+	}
+
+	// Extract domains from HTTP requests
+	result.Domains = extractDomainsFromHTTP(result.HTTPRequests)
+
+	return result, nil
+}
+
 // JSONTranslator implements Translator for generic JSON format
 type JSONTranslator struct {
 	detector *FormatDetector