Skip to content

Commit

Permalink
implemented reprolang tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@matthewnitschke-wk
matthewnitschke-wk committed Sep 15, 2024
1 parent 005eba9 commit b1f45f7
Show file tree
Hide file tree
Showing 11 changed files with 312 additions and 192 deletions.
41 changes: 41 additions & 0 deletions cmd/scip/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,3 +101,44 @@ func TestSCIPSnapshots(t *testing.T) {
return snapshots
})
}

func TestSCIPTests(t *testing.T) {
cwd, err := os.Getwd()
require.Nil(t, err)

testDir := filepath.Join(cwd, "tests", "test_cmd")
testCases, _ := os.ReadDir(testDir)
for _, testCase := range testCases {
if !testCase.IsDir() {
t.Fatalf("not a directory: %v", testCase.Name())
}
t.Run(testCase.Name(), func(t *testing.T) {

baseDirectory := filepath.Join(testDir, testCase.Name())
sources, err := scip.NewSourcesFromDirectory(baseDirectory)
require.Nil(t, err)

index, err := repro.Index("file:/"+baseDirectory, testCase.Name(), sources, []*repro.Dependency{})

passFiles := []string{}
failFiles := []string{}

testFiles, _ := os.ReadDir(baseDirectory)
for _, testFile := range testFiles {
if strings.HasPrefix(testFile.Name(), "passes") {
passFiles = append(passFiles, testFile.Name())
} else if strings.HasPrefix(testFile.Name(), "fails") {
failFiles = append(failFiles, testFile.Name())
} else {
t.Fatalf("Test files must start with 'passes' or 'fails'. Received %v", testFile.Name())
}
}

passRes := ExecuteTestCommand(baseDirectory, passFiles, index, "#")
require.Equal(t, nil, passRes, "Test case %v failed", testCase.Name())

failRes := ExecuteTestCommand(baseDirectory, failFiles, index, "#")
require.NotEqual(t, nil, failRes, "Test case %v failed", testCase.Name())
})
}
}
126 changes: 84 additions & 42 deletions cmd/scip/test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
type testFlags struct {
from string // default: 'index.scip'
commentSyntax string // default: '//'
pathFilters cli.StringSlice
}

func testCommand() cli.Command {
Expand All @@ -29,26 +30,44 @@ on the expected file format of the test files.`,
Flags: []cli.Flag{
fromFlag(&testFlags.from),
commentSyntaxFlag(&testFlags.commentSyntax),
&cli.StringSliceFlag{
Name: "filter",
Aliases: []string{"f"},
Usage: "Filter the test files to run. Only test files that match the provided filter(s).",
Destination: &testFlags.pathFilters,
},
},
Action: func(c *cli.Context) error {
dir := c.Args().Get(0)
return testMain(dir, testFlags)

index, err := readFromOption(testFlags.from)
if err != nil {
return err
}

return ExecuteTestCommand(dir, testFlags.pathFilters.Value(), index, testFlags.commentSyntax)
},
}
return test
}

func testMain(directory string, flags testFlags) error {
index, err := readFromOption(flags.from)
if err != nil {
return err
}

func ExecuteTestCommand(
directory string,
fileFilters []string,
index *scip.Index,
commentSyntax string,
) error {
hasFailure := false

for _, document := range index.Documents {
sourceFilePath := filepath.Join(directory, document.RelativePath)

if len(fileFilters) > 0 {
if !slices.Contains(fileFilters, document.RelativePath) {
continue
}
}

data, err := os.ReadFile(sourceFilePath)
if err != nil {
return err
Expand All @@ -59,7 +78,7 @@ func testMain(directory string, flags testFlags) error {

lines := strings.Split(string(data), "\n")
for lineNumber := 0; lineNumber < len(lines); lineNumber++ {
testCasesAtLine, usedLines := testCasesForLine(lineNumber, lines, flags.commentSyntax)
testCasesAtLine, usedLines := testCasesForLine(lineNumber, lines, commentSyntax)

// if the test file contains no test lines, skip it. Only test the lines
// that the test file dictates should be tested
Expand All @@ -70,7 +89,7 @@ func testMain(directory string, flags testFlags) error {
attributes := attributesForOccurrencesAtLine(lineNumber, document.Occurrences)
for _, testCase := range testCasesAtLine {
filteredAttrs := filterAttributesForTestCase(testCase, attributes)
if !isValidTestCase(testCase, filteredAttrs) {
if !testCase.checkAll(filteredAttrs) {
failures = append(failures, formatFailure(lineNumber, testCase, filteredAttrs))
} else {
successCount++
Expand Down Expand Up @@ -111,7 +130,7 @@ type symbolAttribute struct {
length int

// the type of attribute that this is
kind string
kind symbolAttributeKind

// contextual information about the attribute, as determined
// by the [kind]
Expand All @@ -123,10 +142,34 @@ type symbolAttribute struct {
additionalData []string
}

type symbolAttributeKind string

const (
definitionAttrKind symbolAttributeKind = "definition"
referenceAttrKind symbolAttributeKind = "reference"
forwardDefinitionAttrKind symbolAttributeKind = "forward_definition"
diagnosticAttrKind symbolAttributeKind = "diagnostic"
)

func symbolAttributeKindFromStr(str string) symbolAttributeKind {
switch str {
case "definition":
return definitionAttrKind
case "reference":
return referenceAttrKind
case "forward_definition":
return forwardDefinitionAttrKind
case "diagnostic":
return diagnosticAttrKind
default:
panic(fmt.Sprintf("Unknown symbolAttributeKind: %s", str))
}
}

// symbolAttributeTestCase refers to metadata used to validate
// [symbolAttributes]
type symbolAttributeTestCase struct {
attribute *symbolAttribute
attribute symbolAttribute
enforceLength bool
}

Expand All @@ -135,22 +178,21 @@ type symbolAttributeTestCase struct {
//
// Returns the list of symbolAttributeTestCase(s) for the provided line, and the number of
// of lines that were "consumed" by the cases on this line
func testCasesForLine(lineNumber int, lines []string, commentSyntax string) ([]*symbolAttributeTestCase, int) {
testCases := []*symbolAttributeTestCase{}
func testCasesForLine(lineNumber int, lines []string, commentSyntax string) ([]symbolAttributeTestCase, int) {
testCases := []symbolAttributeTestCase{}

// if the specified lineNumber is outside the bounds of lines
// return an empty array
if lineNumber >= len(lines)-1 {
return testCases, 0
}

testLines := []*symbolAttributeTestCase{}
testLines := []symbolAttributeTestCase{}
usedLines := 0
for i := lineNumber + 1; i < len(lines); i++ {
line := lines[i]

if !strings.HasPrefix(strings.TrimSpace(line), commentSyntax) {
// if the line does not start with a comment, we're done. break
break
}
testCase := parseTestCase(line, lines[i+1:], commentSyntax)
Expand All @@ -163,8 +205,8 @@ func testCasesForLine(lineNumber int, lines []string, commentSyntax string) ([]*
return testLines, usedLines
}

func filterAttributesForTestCase(testCase *symbolAttributeTestCase, attributes []*symbolAttribute) []*symbolAttribute {
filteredAttrs := []*symbolAttribute{}
func filterAttributesForTestCase(testCase symbolAttributeTestCase, attributes []symbolAttribute) []symbolAttribute {
filteredAttrs := []symbolAttribute{}
for _, attr := range attributes {
if testCase.attribute.start >= attr.start && testCase.attribute.start <= (attr.start+attr.length)-1 {
filteredAttrs = append(filteredAttrs, attr)
Expand All @@ -173,22 +215,22 @@ func filterAttributesForTestCase(testCase *symbolAttributeTestCase, attributes [
return filteredAttrs
}

func attributesForOccurrencesAtLine(lineNumber int, occurrences []*scip.Occurrence) []*symbolAttribute {
result := []*symbolAttribute{}
func attributesForOccurrencesAtLine(lineNumber int, occurrences []*scip.Occurrence) []symbolAttribute {
result := []symbolAttribute{}
for _, occ := range occurrences {
if occ.Range[0] == int32(lineNumber) {
pos, _ := scip.NewRange(occ.Range)

start := int(pos.Start.Character)
length := int(pos.End.Character - pos.Start.Character)

kind := "reference"
kind := referenceAttrKind
if scip.SymbolRole_Definition.Matches(occ) {
kind = "definition"
kind = definitionAttrKind
} else if scip.SymbolRole_ForwardDefinition.Matches(occ) {
kind = "forward_definition"
kind = forwardDefinitionAttrKind
}
result = append(result, &symbolAttribute{
result = append(result, symbolAttribute{
start: start,
length: length,
kind: kind,
Expand All @@ -197,10 +239,10 @@ func attributesForOccurrencesAtLine(lineNumber int, occurrences []*scip.Occurren
})

for _, diagnostic := range occ.Diagnostics {
result = append(result, &symbolAttribute{
result = append(result, symbolAttribute{
start: start,
length: length,
kind: "diagnostic",
kind: diagnosticAttrKind,
data: diagnostic.Severity.String(),
additionalData: []string{
diagnostic.Message,
Expand All @@ -212,7 +254,7 @@ func attributesForOccurrencesAtLine(lineNumber int, occurrences []*scip.Occurren
return result
}

func parseTestCase(line string, leadingLines []string, commentSyntax string) *symbolAttributeTestCase {
func parseTestCase(line string, leadingLines []string, commentSyntax string) symbolAttributeTestCase {
start := 0
length := 0
enforceLength := false
Expand Down Expand Up @@ -240,10 +282,10 @@ func parseTestCase(line string, leadingLines []string, commentSyntax string) *sy

// the type of the symbol should be the first word
// this is "definition", "reference", "documentation", "diagnostic", etc..
kind := strings.Split(line, " ")[0]
kindStr := strings.Split(line, " ")[0]

// the data is everything except the type
data := strings.TrimSpace(strings.Replace(line, kind, "", 1))
data := strings.TrimSpace(strings.Replace(line, kindStr, "", 1))

additionalData := []string{}
for i := range leadingLines {
Expand All @@ -266,9 +308,9 @@ func parseTestCase(line string, leadingLines []string, commentSyntax string) *sy
additionalData = append(additionalData, strings.TrimSpace(leadingLine))
}

return &symbolAttributeTestCase{
attribute: &symbolAttribute{
kind: kind,
return symbolAttributeTestCase{
attribute: symbolAttribute{
kind: symbolAttributeKindFromStr(kindStr),
start: start,
length: length,
data: data,
Expand All @@ -278,33 +320,33 @@ func parseTestCase(line string, leadingLines []string, commentSyntax string) *sy
}
}

func isValidTestCase(testCase *symbolAttributeTestCase, attributes []*symbolAttribute) bool {
func (s symbolAttributeTestCase) checkAll(attributes []symbolAttribute) bool {
for _, attr := range attributes {
if isValidTestCaseForAttribute(testCase, attr) {
if s.check(attr) {
return true
}
}
return false
}

func isValidTestCaseForAttribute(testCase *symbolAttributeTestCase, attr *symbolAttribute) bool {
if testCase.enforceLength {
if testCase.attribute.length != attr.length || testCase.attribute.start != attr.start {
func (s symbolAttributeTestCase) check(attr symbolAttribute) bool {
if s.enforceLength {
if s.attribute.length != attr.length || s.attribute.start != attr.start {
return false
}
} else {
if testCase.attribute.start < attr.start || testCase.attribute.start > (attr.start+attr.length)-1 {
if s.attribute.start < attr.start || s.attribute.start > (attr.start+attr.length)-1 {
return false
}
}

if testCase.attribute.kind != attr.kind {
if s.attribute.kind != attr.kind {
return false
}

// check if symbols are equal, a `.` character in the testCaseSymbol is considered
// a wildcard, and matches the correlating group
testCaseSymbolParts := strings.Split(testCase.attribute.data, " ")
testCaseSymbolParts := strings.Split(s.attribute.data, " ")
attrSymbolParts := strings.Split(attr.data, " ")
for i, testCaseSymbolPart := range testCaseSymbolParts {
if testCaseSymbolPart == "." {
Expand All @@ -317,16 +359,16 @@ func isValidTestCaseForAttribute(testCase *symbolAttributeTestCase, attr *symbol

// only validate additionalData if the testCases provides one
// otherwise, ignore what the attribute specifies
if len(testCase.attribute.additionalData) > 0 {
if !slices.Equal(testCase.attribute.additionalData, attr.additionalData) {
if len(s.attribute.additionalData) > 0 {
if !slices.Equal(s.attribute.additionalData, attr.additionalData) {
return false
}
}

return true
}

func formatFailure(lineNumber int, testCase *symbolAttributeTestCase, attributesAtLine []*symbolAttribute) string {
func formatFailure(lineNumber int, testCase symbolAttributeTestCase, attributesAtLine []symbolAttribute) string {
failureDesc := []string{
fmt.Sprintf("Failure - row: %d, column: %d", lineNumber, testCase.attribute.start),
fmt.Sprintf(" Expected: '%s %s'", testCase.attribute.kind, testCase.attribute.data),
Expand Down
Loading

0 comments on commit b1f45f7

Please sign in to comment.