Azure AI in Production: RAG Architecture, Cost Optimization, and Enterprise Patterns That Actually Work

              public class ProductionRagService
{
    private readonly AzureOpenAIClient _openAiClient;
    private readonly SearchClient _searchClient;
    private readonly ILogger _logger;

    // CRITICAL: Cache embeddings to avoid redundant API calls
    private readonly IMemoryCache _embeddingCache;

    public async Task<RagResponse> AnswerQuestionAsync(
        string question,
        CancellationToken ct = default)
    {
        // 1. Generate query embedding (with caching)
        var queryEmbedding = await GetOrCreateEmbeddingAsync(
            question, ct);

        // 2. Hybrid search: vector + keyword + semantic reranking
        var searchResults = await _searchClient.SearchAsync<
            DocumentSearchResult>(
            new SearchOptions
            {
                VectorSearch = new VectorSearchOptions
                {
                    Queries = {
                        new VectorizedQuery(queryEmbedding)
                        {
                            KNearestNeighborsCount = 10,
                            Fields = { "contentVector" }
                        }
                    }
                },
                QueryType = SearchQueryType.Semantic,
                SemanticSearch = new SemanticSearchOptions
                {
                    SemanticConfigurationName = "default",
                    QueryCaption = new QueryCaption(
                        QueryCaptionType.Extractive)
                },
                Top = 5
            }, ct);

        // 3. CRITICAL: Filter by relevance score to avoid injecting noise
        var relevantDocs = searchResults.Value.GetResults()
            .Where(r => r.Score > 0.7) // Threshold based on your data
            .Take(3) // Limit context window usage
            .ToList();

        if (!relevantDocs.Any())
        {
            return new RagResponse(
                "I don't have enough information to answer that question.",
                sources: Array.Empty<string>());
        }

        // 4. Build context with source attribution
        var context = string.Join("

",
            relevantDocs.Select((doc, i) =>
                $"[Source {i+1}]: {doc.Document.Content}"));

        // 5. CRITICAL: Use system prompt to enforce grounding
        var systemPrompt = $@"You are a helpful assistant that answers questions based ONLY on the provided context.
If the answer is not in the context, say "I don't know" rather than making up information.
Always cite your sources using [Source N] format.

Context:
{context}";

        // 6. Generate response with streaming for better UX
        var chatClient = _openAiClient.GetChatClient("gpt-4o");
        var messages = new List<ChatMessage>
        {
            new SystemChatMessage(systemPrompt),
            new UserChatMessage(question)
        };

        var response = await chatClient.CompleteChatAsync(
            messages, ct);

        return new RagResponse(
            response.Value.Content[0].Text,
            sources: relevantDocs.Select(d => d.Document.SourceUrl).ToArray());
    }

    private async Task<float[]> GetOrCreateEmbeddingAsync(
        string text, CancellationToken ct)
    {
        // Cache embeddings for 24 hours to reduce API costs
        if (_embeddingCache.TryGetValue(text, out float[] cached))
            return cached;

        var embeddingClient = _openAiClient.GetEmbeddingClient(
            "text-embedding-3-small"); // Cheaper than ada-002
        var result = await embeddingClient.GenerateEmbeddingAsync(
            text, cancellationToken: ct);

        _embeddingCache.Set(text, result.Value.Vector,
            TimeSpan.FromHours(24));

        return result.Value.Vector;
    }
}

            

              public class TokenBudgetMiddleware
{
    private readonly IDistributedCache _cache;
    private readonly int _maxTokensPerUserPerDay = 100000;

    public async Task<bool> CheckTokenBudgetAsync(
        string userId,
        int estimatedTokens)
    {
        var key = $"token_budget:{userId}:{DateTime.UtcNow:yyyy-MM-dd}";
        var currentUsage = await _cache.GetIntAsync(key) ?? 0;

        if (currentUsage + estimatedTokens > _maxTokensPerUserPerDay)
        {
            return false; // Budget exceeded
        }

        await _cache.SetIntAsync(
            key,
            currentUsage + estimatedTokens,
            new DistributedCacheEntryOptions
            {
                AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1)
            });

        return true;
    }
}

            

              public class SecureRagService
{
    private readonly ContentSafetyClient _contentSafety;

    public async Task<RagResponse> SafeAnswerAsync(
        string userQuestion, CancellationToken ct)
    {
        // 1. Check input for malicious content
        var inputAnalysis = await _contentSafety.AnalyzeTextAsync(
            new TextAnalysisInput(userQuestion), ct);

        if (inputAnalysis.Value.CategoriesAnalysis.Any(
            c => c.Severity > 0)) // Any harmful content detected
        {
            _logger.LogWarning(
                "Blocked prompt injection attempt: {Question}",
                userQuestion);
            return new RagResponse(
                "I cannot process that request.");
        }

        // 2. Proceed with RAG pipeline...
        var ragResponse = await _ragPipeline.AnswerAsync(
            userQuestion, ct);

        // 3. CRITICAL: Validate output before returning to user
        var outputAnalysis = await _contentSafety.AnalyzeTextAsync(
            new TextAnalysisInput(ragResponse.Answer), ct);

        if (outputAnalysis.Value.CategoriesAnalysis.Any(
            c => c.Severity > 2)) // High severity
        {
            return new RagResponse(
                "I apologize, but I cannot provide that information.");
        }

        return ragResponse;
    }
}

            

              var index = new SearchIndex("documents", new[]
{
    new SimpleField("id", SearchFieldDataType.String)
    {
        IsKey = true,
        IsFilterable = true
    },
    new SearchableField("content")
    {
        IsSearchable = true,
        AnalyzerName = LexicalAnalyzerName.EnMicrosoft // Better than standard
    },
    new SearchableField("title")
    {
        IsSearchable = true,
        AnalyzerName = LexicalAnalyzerName.EnMicrosoft
    },
    new SimpleField("sourceUrl", SearchFieldDataType.String)
    {
        IsFilterable = true,
        IsRetrievable = true
    },
    new SimpleField("lastModified", SearchFieldDataType.DateTimeOffset)
    {
        IsFilterable = true,
        IsSortable = true
    },
    // CRITICAL: Vector field for embeddings
    new VectorSearchField(
        "contentVector",
        1536, // Dimensions for text-embedding-3-small
        new HnswAlgorithmConfiguration("hnsw")
        {
            Parameters = new HnswParameters
            {
                M = 4,
                EfConstruction = 200,
                EfSearch = 200,
                Metric = VectorSearchAlgorithmMetric.Cosine
            }
        })
});

// Enable semantic ranking
index.SemanticSettings = new SemanticSettings
{
    Configurations = {
        new SemanticConfiguration("default", new()
        {
            PrioritizedFields = new()
            {
                TitleField = new SemanticField("title"),
                ContentFields = { new SemanticField("content") }
            }
        })
    }
};

await searchClient.CreateIndexAsync(index);

            

              public async Task<DocumentAnalysisResult> ProcessDocumentAsync(
    Stream documentStream,
    string documentType,
    CancellationToken ct)
{
    // 1. Check if we've already processed this document
    var documentHash = await ComputeSha256HashAsync(documentStream);
    var cacheKey = $"doc_analysis:{documentHash}";

    if (_cache.TryGetValue(cacheKey, out DocumentAnalysisResult cached))
        return cached;

    // 2. Use prebuilt model for common types, custom for specialized
    var modelId = documentType switch
    {
        "invoice" => "prebuilt-invoice",
        "receipt" => "prebuilt-receipt",
        "contract" => "custom-contract-model-v2", // Trained custom model
        _ => "prebuilt-document"
    };

    // 3. Use async API for documents >10 pages
    var operation = await _documentAnalysisClient
        .AnalyzeDocumentAsync(
            WaitUntil.Started,
            modelId,
            documentStream,
            cancellationToken: ct);

    // 4. Poll for completion with exponential backoff
    var result = await operation.WaitForCompletionAsync(ct);

    // 5. Cache for 30 days (documents rarely change)
    _cache.Set(cacheKey, result.Value, TimeSpan.FromDays(30));

    return result.Value;
}

            

              public class AiTelemetryService
{
    private readonly TelemetryClient _telemetry;

    public void TrackAiRequest(
        string model,
        string operation,
        int inputTokens,
        int outputTokens,
        double latencyMs,
        bool success,
        string? errorCode = null)
    {
        var metrics = new Dictionary<string, double>
        {
            ["input_tokens"] = inputTokens,
            ["output_tokens"] = outputTokens,
            ["total_tokens"] = inputTokens + outputTokens,
            ["latency_ms"] = latencyMs,
            ["cost_usd"] = CalculateCost(model, inputTokens, outputTokens)
        };

        _telemetry.TrackEvent(
            $"AI_{operation}",
            new Dictionary<string, string>
            {
                ["model"] = model,
                ["success"] = success.ToString(),
                ["error_code"] = errorCode ?? "none"
            },
            metrics);
    }

    private double CalculateCost(
        string model, int inputTokens, int outputTokens)
    {
        // Azure OpenAI pricing as of July 2026
        return model switch
        {
            "gpt-4o" => (inputTokens * 0.0000025) + (outputTokens * 0.00001),
            "gpt-4o-mini" => (inputTokens * 0.00000015) + (outputTokens * 0.0000006),
            "text-embedding-3-small" => inputTokens * 0.00000002,
            _ => 0
        };
    }
}

            

              public class ResilientOpenAiClient
{
    private readonly List<AzureOpenAIClient> _clients;
    private readonly AsyncPolicy _retryPolicy;
    private int _currentClientIndex = 0;

    public ResilientOpenAiClient(
        IEnumerable<AzureOpenAIClient> clients)
    {
        _clients = clients.ToList();

        // CRITICAL: Retry with exponential backoff for 429 and 500 errors
        _retryPolicy = Policy
            .Handle<RequestFailedException>(ex =>
                ex.Status == 429 || ex.Status >= 500)
            .WaitAndRetryAsync(
                retryCount: 3,
                sleepDurationProvider: attempt =>
                    TimeSpan.FromSeconds(Math.Pow(2, attempt)),
                onRetry: (ex, delay, attempt, ctx) =>
                {
                    _logger.LogWarning(
                        "Retry {Attempt} after {Delay}ms due to {Status}",
                        attempt, delay.TotalMilliseconds, ex.Status);
                });
    }

    public async Task<ChatCompletion> CompleteChatAsync(
        ChatMessage[] messages, CancellationToken ct)
    {
        return await _retryPolicy.ExecuteAsync(async () =>
        {
            // Round-robin across multiple deployments
            var client = _clients[
                Interlocked.Increment(ref _currentClientIndex)
                % _clients.Count];

            var chatClient = client.GetChatClient("gpt-4o");
            return await chatClient.CompleteChatAsync(
                messages, ct);
        });
    }
}