Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,15 @@ private static List<FeedbackItem> SampleItems() =>
new FeedbackItem { Text = "Rename FooClient to BarClient" }
];

private static FeedbackClassificationTemplate CreateTemplate(EditScope editScope) =>
private static FeedbackClassificationTemplate CreateTemplate(EditScope editScope, bool specInspectionAvailable = true) =>
new(
serviceName: "Widget",
language: "csharp",
referenceDocContent: "reference",
items: SampleItems(),
globalContext: "ctx",
editScope: editScope);
editScope: editScope,
specInspectionAvailable: specInspectionAvailable);

[Test]
public void AllScope_EmitsNoEditScopeBias()
Expand Down Expand Up @@ -53,4 +54,16 @@ public void SpecInputsScope_BiasesTowardTspApplicable()
Assert.That(prompt, Does.Contain("classify it as **TSP_APPLICABLE**"));
Assert.That(prompt, Does.Not.Contain("EDIT SCOPE — CUSTOM CODE ONLY"));
}

[Test]
public void CustomCodeScopeWithoutSpecInspection_DoesNotAdvertiseSpecTools()
{
var prompt = CreateTemplate(EditScope.CustomCode, specInspectionAvailable: false).BuildPrompt();

Assert.That(prompt, Does.Contain("TypeSpec inspection unavailable"));
Assert.That(prompt, Does.Not.Contain("**Available Tools:**"));
Assert.That(prompt, Does.Not.Contain("Always search the TypeSpec files first"));
Assert.That(prompt, Does.Not.Contain("reference documentation below"));
Assert.That(prompt, Does.Not.Contain("Consult the reference documentation provided"));
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using System.Reflection;
using Azure.Sdk.Tools.Cli.CopilotAgents;
using Azure.Sdk.Tools.Cli.Helpers;
using Azure.Sdk.Tools.Cli.Models;
Expand All @@ -10,8 +11,6 @@
using Microsoft.Extensions.Logging;
using Moq;
using NUnit.Framework;
using System.Reflection;

using static Azure.Sdk.Tools.Cli.Tests.TestHelpers.TestCategories;

namespace Azure.Sdk.Tools.Cli.Tests.Services;
Expand Down Expand Up @@ -256,6 +255,57 @@ public async Task ClassifyAsync_ItemsStillTspApplicable_ReturnsFalse()
Assert.That(item1.Status, Is.EqualTo(FeedbackStatus.TSP_APPLICABLE));
}

[Test]
public async Task ClassifyAsync_CustomCodeScopeWithoutTspPath_DoesNotAccessSpecRepo()
{
var service = CreateMockedService();
var item = CreateTestItem("error CS0103: The name 'BlobUriForRepairTest' does not exist", "item-1");
var items = new List<FeedbackItem> { item };
var batchResponse = """
[item-1]
Classification: CODE_CUSTOMIZATION
Reason: The failure is in a custom partial class and can be fixed without changing TypeSpec inputs
""";

_mockAgentRunner
.Setup(x => x.RunAsync(It.IsAny<CopilotAgent<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(batchResponse);

await service.ClassifyItemsAsync(
items,
globalContext: "Custom-code build repair",
tspProjectPath: null,
language: "csharp",
serviceName: "ContentSafety",
editScope: EditScope.CustomCode);

Assert.That(item.Status, Is.EqualTo(FeedbackStatus.CODE_CUSTOMIZATION));
_mockTypeSpecHelper.Verify(
x => x.GetSpecRepoRootPath(It.IsAny<string>()),
Times.Never);
_mockAgentRunner.Verify(
x => x.RunAsync(It.IsAny<CopilotAgent<string>>(), It.IsAny<CancellationToken>()),
Times.Once);
}

[TestCase(EditScope.SpecInputs)]
[TestCase(EditScope.All)]
public void ClassifyAsync_SpecInputScopeWithoutTspPath_ThrowsArgumentException(EditScope editScope)
{
var service = CreateMockedService();
var items = new List<FeedbackItem> { CreateTestItem("Rename FooClient", "item-1") };

Assert.ThrowsAsync<ArgumentException>(() => service.ClassifyItemsAsync(
items,
globalContext: string.Empty,
tspProjectPath: null,
editScope: editScope));

_mockAgentRunner.Verify(
x => x.RunAsync(It.IsAny<CopilotAgent<string>>(), It.IsAny<CancellationToken>()),
Times.Never);
}

/// <summary>
/// Tests that REQUIRES_MANUAL_INTERVENTION classification is applied correctly.
/// Note: NextAction guidance generation was removed - REQUIRES_MANUAL_INTERVENTION items are just classified.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public class FeedbackClassificationTemplate : BasePromptTemplate
private readonly List<FeedbackItem> _items;
private readonly string _globalContext;
private readonly EditScope _editScope;
private readonly bool _specInspectionAvailable;

/// <summary>
/// Initializes a new batch classification template.
Expand All @@ -37,20 +38,23 @@ public class FeedbackClassificationTemplate : BasePromptTemplate
/// is asked to prefer the in-scope axis for items that could be addressed either way (e.g. a rename
/// achievable in spec inputs OR custom code), so fixable items are not reported as out of scope.
/// </param>
/// <param name="specInspectionAvailable">Whether tools and reference content for a local TypeSpec project are available.</param>
public FeedbackClassificationTemplate(
string? serviceName,
string language,
string referenceDocContent,
List<FeedbackItem> items,
string globalContext,
EditScope editScope = EditScope.All)
EditScope editScope = EditScope.All,
bool specInspectionAvailable = true)
{
_serviceName = serviceName;
_language = language;
_referenceDocContent = referenceDocContent;
_items = items;
_globalContext = globalContext;
_editScope = editScope;
_specInspectionAvailable = specInspectionAvailable;
}

public override string BuildPrompt()
Expand All @@ -59,7 +63,7 @@ public override string BuildPrompt()
var constraints = BuildClassificationConditions();
var examples = BuildExamples();
var outputRequirements = BuildOutputRequirements();

return BuildStructuredPrompt(taskInstructions, constraints, examples, outputRequirements);
}

Expand All @@ -82,6 +86,10 @@ protected override string BuildSystemRole()
private string BuildTaskInstructions()
{
var sb = new StringBuilder();
var tspApplicabilityBasis = _specInspectionAvailable
? "based on the reference documentation below"
: "based on known TypeSpec capabilities and the available feedback context";

sb.AppendLine($"""
**Current Context:**
- Service: {_serviceName ?? "N/A"}
Expand All @@ -95,19 +103,36 @@ private string BuildTaskInstructions()
sb.AppendLine(scopeGuidance);
}

sb.AppendLine($"""
sb.AppendLine("""

**Task:**
Classify ALL of the feedback items listed below. For each item, determine the appropriate classification: **TSP_APPLICABLE**, **CODE_CUSTOMIZATION**, **SUCCESS**, or **REQUIRES_MANUAL_INTERVENTION**.
- If the feedback is non-actionable (discussion, informational, "keep as is", or about build/generation succeeding), classify as **SUCCESS**.
- If the feedback is actionable AND TypeSpec client customization decorators can address it (based on the reference documentation below), classify as **TSP_APPLICABLE**.
- If the feedback is actionable AND TypeSpec client customization decorators can address it ({tspApplicabilityBasis}), classify as **TSP_APPLICABLE**.
- If the feedback is actionable, TypeSpec decorators CANNOT address it, but automated code patching could fix it (e.g., compile errors from method signature changes, parameter additions/removals, symbol renames in generated code), classify as **CODE_CUSTOMIZATION**. Include specific repair instructions in the Reason.
Comment thread
JoshLove-msft marked this conversation as resolved.
- If the feedback is actionable but requires complex manual work that cannot be automated (e.g., new feature implementation, architectural changes, custom business logic), classify as **REQUIRES_MANUAL_INTERVENTION**.
""");

if (_specInspectionAvailable)
{
sb.AppendLine("""

**IMPORTANT — Check for already-applied customizations:**
Before classifying any item as TSP_APPLICABLE, use the available tools to search the TypeSpec project files (e.g., `client.tsp`, `customizations.tsp`) for the decorator or customization that would address the feedback. If the customization is already present in the TypeSpec files, classify the item as **SUCCESS** (the change has already been applied). Use `grep_search` or `read_file` to verify.
**IMPORTANT — Check for already-applied customizations:**
Before classifying any item as TSP_APPLICABLE, use the available tools to search the TypeSpec project files (e.g., `client.tsp`, `customizations.tsp`) for the decorator or customization that would address the feedback. If the customization is already present in the TypeSpec files, classify the item as **SUCCESS** (the change has already been applied). Use `grep_search` or `read_file` to verify.

Use the available tools to inspect the TypeSpec project files when needed to determine if decorators are applicable.
Use the available tools to inspect the TypeSpec project files when needed to determine if decorators are applicable.
""");
}
else
{
sb.AppendLine("""

**TypeSpec inspection unavailable:**
No local TypeSpec checkout or spec-repo tools are available in this custom-code-only run. Do not attempt to inspect TypeSpec files. Classify using the feedback, build context, and edit-scope guidance.
""");
}

sb.AppendLine($"""

**Global Context:**
{_globalContext}
Expand All @@ -129,20 +154,23 @@ private string BuildTaskInstructions()
""");
}

sb.AppendLine($"""
if (_specInspectionAvailable)
{
sb.AppendLine($"""

---
---

**TypeSpec Client Customizations Reference:**
<reference_doc>
{_referenceDocContent}
</reference_doc>
**TypeSpec Client Customizations Reference:**
<reference_doc>
{_referenceDocContent}
</reference_doc>

**Available Tools:**
- `read_file`: Read contents of specific files in the spec repo
- `list_dir`: List files and directories
- `grep_search`: Search for patterns within files
""");
**Available Tools:**
- `read_file`: Read contents of specific files in the spec repo
- `list_dir`: List files and directories
- `grep_search`: Search for patterns within files
""");
}

return sb.ToString();
}
Expand Down Expand Up @@ -199,7 +227,34 @@ CODE_CUSTOMIZATION sparingly and only when unavoidable.

private string BuildClassificationConditions()
{
return """
var specInspectionGuidance = _specInspectionAvailable
? """
- Actionable AND a TypeSpec decorator from the reference doc could address it → **check the TypeSpec files first** using `grep_search` or `read_file`
- If the decorator/customization is already present in the TypeSpec files → **SUCCESS** (already applied)
- If not present → **TSP_APPLICABLE**
"""
: """
- Actionable AND a TypeSpec decorator could address it → apply the edit-scope guidance without attempting to inspect unavailable TypeSpec files
""";

var tspVerificationGuidance = _specInspectionAvailable
? """
**Always search the TypeSpec files first** (using `grep_search` or `read_file`) to confirm the decorator
is NOT already present before classifying as TSP_APPLICABLE. If it is already present, classify as SUCCESS.
"""
: "Local TypeSpec files are unavailable in this run; do not attempt to use spec-repo inspection tools.";

var tspApplicabilityGuidance = _specInspectionAvailable
? """
Consult the reference documentation provided to determine if any supported
TypeSpec client customization decorator can address the feedback.
"""
: """
Use known TypeSpec client customization capabilities and the available feedback context
to determine whether the change genuinely requires a spec-input edit.
""";

return $$"""
**Decision Logic (apply to EACH item independently):**

**If Context is NON-EMPTY** (check first):
Expand All @@ -209,9 +264,7 @@ private string BuildClassificationConditions()

**If Context is EMPTY** (first attempt):
- Non-actionable (informational, "keep as is", past tense, build success, discussion, question) → **SUCCESS**
- Actionable AND a TypeSpec decorator from the reference doc could address it → **check the TypeSpec files first** using `grep_search` or `read_file`
- If the decorator/customization is already present in the TypeSpec files → **SUCCESS** (already applied)
- If not present → **TSP_APPLICABLE**
{{specInspectionGuidance}}
- Actionable, no TypeSpec decorator applies, but automated patching can fix (compile errors, signature changes, parameter additions/removals, symbol renames, linting or typing errors) → **CODE_CUSTOMIZATION**
- Actionable BUT requires complex manual implementation → **REQUIRES_MANUAL_INTERVENTION**

Expand All @@ -222,12 +275,10 @@ private string BuildClassificationConditions()
- Informational: Explanations, questions, acknowledgments
- Build/generation success with no errors
- Discussion or questions without a clear directive

**TypeSpec Decorator Applicability (TSP_APPLICABLE):**
Consult the reference documentation provided to determine if any supported
TypeSpec client customization decorator can address the feedback.
**Always search the TypeSpec files first** (using `grep_search` or `read_file`) to confirm the decorator
is NOT already present before classifying as TSP_APPLICABLE. If it is already present, classify as SUCCESS.
**TypeSpec Decorator Applicability (TSP_APPLICABLE):**
{{tspApplicabilityGuidance}}
{{tspVerificationGuidance}}
Comment thread
JoshLove-msft marked this conversation as resolved.

**Common feedback patterns that ARE TypeSpec-applicable:**
- Renaming (client, operation, model, property, enum value) → `@@clientName` or `@clientName`
Expand Down
Loading
Loading