get language from .gitattributes file

modules/git/repo_attribute.go

@@ -15,6 +15,7 @@ type CheckAttributeOpts struct {
 	AllAttributes bool
 	Attributes    []string
 	Filenames     []string
+	IndexFile     string
 }
 
 // CheckAttribute return the Blame object of file
@@ -54,7 +55,12 @@ func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[
 
 	cmd := NewCommand(cmdArgs...)
 
-	if err := cmd.RunInDirPipeline(repo.Path, stdOut, stdErr); err != nil {
+	env := make([]string, 0, 1)
+	if len(opts.IndexFile) > 0 {
+		env = append(env, "GIT_INDEX_FILE="+opts.IndexFile)
+	}
+
+	if err := cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdOut, stdErr, nil); err != nil {
 		return nil, fmt.Errorf("Failed to run check-attr: %v\n%s\n%s", err, stdOut.String(), stdErr.String())
 	}
 

modules/git/repo_language_stats_gogit.go

@@ -20,7 +20,7 @@ import (
 )
 
 // GetLanguageStats calculates language stats for git repository at specified commit
-func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
+func (repo *Repository) GetLanguageStats(commitID string, preCheck func(path string) (string, bool)) (map[string]int64, error) {
 	r, err := git.PlainOpen(repo.Path)
 	if err != nil {
 		return nil, err
@@ -57,9 +57,18 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			return nil
 		}
 
-		// TODO: Use .gitattributes file for linguist overrides
+		language := ""
+		skip := false
+		if preCheck != nil {
+			language, skip = preCheck(f.Name)
+			if skip {
+				return nil
+			}
+		}
 
-		language := analyze.GetCodeLanguage(f.Name, content)
+		if len(language) == 0 {
+			language = analyze.GetCodeLanguage(f.Name, content)
+		}
 		if language == enry.OtherLanguage || language == "" {
 			return nil
 		}

modules/git/repo_language_stats_nogogit.go

@@ -19,7 +19,7 @@ import (
 )
 
 // GetLanguageStats calculates language stats for git repository at specified commit
-func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
+func (repo *Repository) GetLanguageStats(commitID string, preCheck func(path string) (string, bool)) (map[string]int64, error) {
 	// We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary.
 	// so let's create a batch stdin and stdout
 
@@ -128,10 +128,22 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			continue
 		}
 
-		// TODO: Use .gitattributes file for linguist overrides
-		// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
-		// - eg. do the all the detection tests using filename first before reading content.
-		language := analyze.GetCodeLanguage(f.Name(), content)
+		// Use .gitattributes file for linguist overrides
+		language := ""
+		skip := false
+		if preCheck != nil {
+			language, skip = preCheck(f.Name())
+			if skip {
+				continue
+			}
+		}
+
+		if len(language) == 0 {
+			// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
+			// - eg. do the all the detection tests using filename first before reading content.
+			language = analyze.GetCodeLanguage(f.Name(), content)
+		}
+
 		if language == enry.OtherLanguage || language == "" {
 			continue
 		}

modules/indexer/stats/db.go

@@ -5,9 +5,13 @@
 package stats
 
 import (
+	"io/ioutil"
+	"os"
+
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
 )
 
 // DBIndexer implements Indexer interface to use database's like search
@@ -47,8 +51,62 @@ func (db *DBIndexer) Index(id int64) error {
 		return nil
 	}
 
+	var tmpIndex *os.File
+	if git.CheckGitVersionAtLeast("1.7.8") == nil {
+		tmpIndex, err = ioutil.TempFile("", "index")
+		if err != nil {
+			return err
+		}
+		defer func() {
+			err := util.Remove(tmpIndex.Name())
+			if err != nil {
+				log.Error("failed to remove tmp index file: %v", err)
+			}
+		}()
+
+		_, err = git.NewCommand("read-tree", commitID).
+			RunInDirWithEnv(gitRepo.Path, []string{"GIT_INDEX_FILE=" + tmpIndex.Name()})
+		if err != nil {
+			return err
+		}
+	}
+
 	// Calculate and save language statistics to database
-	stats, err := gitRepo.GetLanguageStats(commitID)
+	stats, err := gitRepo.GetLanguageStats(commitID, func(path string) (string, bool) {
+		// get language follow linguist rulers
+		// linguist-language=<lang> attribute to an language
+		// linguist-vendored attribute to vendor or un-vendor paths
+
+		if tmpIndex == nil {
+			return "", false
+		}
+
+		name2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{
+			Attributes: []string{"linguist-vendored", "linguist-language"},
+			Filenames:  []string{path},
+			CachedOnly: true,
+			IndexFile:  tmpIndex.Name(),
+		})
+		if err != nil {
+			log.Error("gitRepo.CheckAttribute: %v", err)
+			return "", false
+		}
+
+		attribute2info, has := name2attribute2info[path]
+		if !has {
+			return "", false
+		}
+		if attribute2info["linguist-vendored"] == "set" {
+			return "", true
+		}
+
+		lang := attribute2info["linguist-language"]
+		if lang == "unspecified" {
+			lang = ""
+		}
+
+		return lang, false
+	})
 	if err != nil {
 		log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
 		return err