2021-06-14 22:50:43 +05:30
// Copyright 2021 The Gitea Authors. All rights reserved.
2022-11-27 23:50:29 +05:30
// SPDX-License-Identifier: MIT
2021-06-14 22:50:43 +05:30
package mirror
import (
"context"
"fmt"
"strings"
"time"
2022-05-20 19:38:52 +05:30
"code.gitea.io/gitea/models/db"
2021-12-10 06:57:50 +05:30
repo_model "code.gitea.io/gitea/models/repo"
2022-10-17 04:59:26 +05:30
system_model "code.gitea.io/gitea/models/system"
2021-06-14 22:50:43 +05:30
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
2021-12-01 01:36:32 +05:30
"code.gitea.io/gitea/modules/process"
2021-06-14 22:50:43 +05:30
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
)
// gitShortEmptySha Git short empty SHA
const gitShortEmptySha = "0000000"
// UpdateAddress writes new address to Git repository and database
2022-01-20 04:56:57 +05:30
func UpdateAddress ( ctx context . Context , m * repo_model . Mirror , addr string ) error {
2021-06-14 22:50:43 +05:30
remoteName := m . GetRemoteName ( )
2022-06-02 22:26:32 +05:30
repoPath := m . GetRepository ( ) . RepoPath ( )
2021-06-14 22:50:43 +05:30
// Remove old remote
2022-10-23 20:14:45 +05:30
_ , _ , err := git . NewCommand ( ctx , "remote" , "rm" ) . AddDynamicArguments ( remoteName ) . RunStdString ( & git . RunOpts { Dir : repoPath } )
2021-06-14 22:50:43 +05:30
if err != nil && ! strings . HasPrefix ( err . Error ( ) , "exit status 128 - fatal: No such remote " ) {
return err
}
2022-10-23 20:14:45 +05:30
cmd := git . NewCommand ( ctx , "remote" , "add" ) . AddDynamicArguments ( remoteName ) . AddArguments ( "--mirror=fetch" ) . AddDynamicArguments ( addr )
2022-03-27 17:24:09 +05:30
if strings . Contains ( addr , "://" ) && strings . Contains ( addr , "@" ) {
2022-03-31 07:55:40 +05:30
cmd . SetDescription ( fmt . Sprintf ( "remote add %s --mirror=fetch %s [repo_path: %s]" , remoteName , util . SanitizeCredentialURLs ( addr ) , repoPath ) )
2022-03-27 17:24:09 +05:30
} else {
cmd . SetDescription ( fmt . Sprintf ( "remote add %s --mirror=fetch %s [repo_path: %s]" , remoteName , addr , repoPath ) )
}
2022-04-01 08:25:30 +05:30
_ , _ , err = cmd . RunStdString ( & git . RunOpts { Dir : repoPath } )
2021-06-14 22:50:43 +05:30
if err != nil && ! strings . HasPrefix ( err . Error ( ) , "exit status 128 - fatal: No such remote " ) {
return err
}
if m . Repo . HasWiki ( ) {
wikiPath := m . Repo . WikiPath ( )
2022-01-20 04:56:57 +05:30
wikiRemotePath := repo_module . WikiRemoteURL ( ctx , addr )
2021-06-14 22:50:43 +05:30
// Remove old remote of wiki
2022-10-23 20:14:45 +05:30
_ , _ , err = git . NewCommand ( ctx , "remote" , "rm" ) . AddDynamicArguments ( remoteName ) . RunStdString ( & git . RunOpts { Dir : wikiPath } )
2021-06-14 22:50:43 +05:30
if err != nil && ! strings . HasPrefix ( err . Error ( ) , "exit status 128 - fatal: No such remote " ) {
return err
}
2022-10-23 20:14:45 +05:30
cmd = git . NewCommand ( ctx , "remote" , "add" ) . AddDynamicArguments ( remoteName ) . AddArguments ( "--mirror=fetch" ) . AddDynamicArguments ( wikiRemotePath )
2022-03-27 17:24:09 +05:30
if strings . Contains ( wikiRemotePath , "://" ) && strings . Contains ( wikiRemotePath , "@" ) {
2022-03-31 07:55:40 +05:30
cmd . SetDescription ( fmt . Sprintf ( "remote add %s --mirror=fetch %s [repo_path: %s]" , remoteName , util . SanitizeCredentialURLs ( wikiRemotePath ) , wikiPath ) )
2022-03-27 17:24:09 +05:30
} else {
cmd . SetDescription ( fmt . Sprintf ( "remote add %s --mirror=fetch %s [repo_path: %s]" , remoteName , wikiRemotePath , wikiPath ) )
}
2022-04-01 08:25:30 +05:30
_ , _ , err = cmd . RunStdString ( & git . RunOpts { Dir : wikiPath } )
2021-06-14 22:50:43 +05:30
if err != nil && ! strings . HasPrefix ( err . Error ( ) , "exit status 128 - fatal: No such remote " ) {
return err
}
}
m . Repo . OriginalURL = addr
2022-05-20 19:38:52 +05:30
return repo_model . UpdateRepositoryCols ( ctx , m . Repo , "original_url" )
2021-06-14 22:50:43 +05:30
}
// mirrorSyncResult contains information of a updated reference.
// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
type mirrorSyncResult struct {
refName string
oldCommitID string
newCommitID string
}
// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
func parseRemoteUpdateOutput ( output string ) [ ] * mirrorSyncResult {
results := make ( [ ] * mirrorSyncResult , 0 , 3 )
lines := strings . Split ( output , "\n" )
for i := range lines {
// Make sure reference name is presented before continue
idx := strings . Index ( lines [ i ] , "-> " )
if idx == - 1 {
continue
}
refName := lines [ i ] [ idx + 3 : ]
switch {
case strings . HasPrefix ( lines [ i ] , " * " ) : // New reference
if strings . HasPrefix ( lines [ i ] , " * [new tag]" ) {
refName = git . TagPrefix + refName
} else if strings . HasPrefix ( lines [ i ] , " * [new branch]" ) {
refName = git . BranchPrefix + refName
}
results = append ( results , & mirrorSyncResult {
refName : refName ,
oldCommitID : gitShortEmptySha ,
} )
case strings . HasPrefix ( lines [ i ] , " - " ) : // Delete reference
results = append ( results , & mirrorSyncResult {
refName : refName ,
newCommitID : gitShortEmptySha ,
} )
case strings . HasPrefix ( lines [ i ] , " + " ) : // Force update
if idx := strings . Index ( refName , " " ) ; idx > - 1 {
refName = refName [ : idx ]
}
delimIdx := strings . Index ( lines [ i ] [ 3 : ] , " " )
if delimIdx == - 1 {
log . Error ( "SHA delimiter not found: %q" , lines [ i ] )
continue
}
shas := strings . Split ( lines [ i ] [ 3 : delimIdx + 3 ] , "..." )
if len ( shas ) != 2 {
log . Error ( "Expect two SHAs but not what found: %q" , lines [ i ] )
continue
}
results = append ( results , & mirrorSyncResult {
refName : refName ,
oldCommitID : shas [ 0 ] ,
newCommitID : shas [ 1 ] ,
} )
case strings . HasPrefix ( lines [ i ] , " " ) : // New commits of a reference
delimIdx := strings . Index ( lines [ i ] [ 3 : ] , " " )
if delimIdx == - 1 {
log . Error ( "SHA delimiter not found: %q" , lines [ i ] )
continue
}
shas := strings . Split ( lines [ i ] [ 3 : delimIdx + 3 ] , ".." )
if len ( shas ) != 2 {
log . Error ( "Expect two SHAs but not what found: %q" , lines [ i ] )
continue
}
results = append ( results , & mirrorSyncResult {
refName : refName ,
oldCommitID : shas [ 0 ] ,
newCommitID : shas [ 1 ] ,
} )
default :
log . Warn ( "parseRemoteUpdateOutput: unexpected update line %q" , lines [ i ] )
}
}
return results
}
2021-10-21 14:15:25 +05:30
func pruneBrokenReferences ( ctx context . Context ,
2021-12-10 06:57:50 +05:30
m * repo_model . Mirror ,
2021-10-21 14:15:25 +05:30
repoPath string ,
timeout time . Duration ,
stdoutBuilder , stderrBuilder * strings . Builder ,
2022-01-20 23:16:10 +05:30
isWiki bool ,
) error {
2021-10-21 14:15:25 +05:30
wiki := ""
if isWiki {
wiki = "Wiki "
}
stderrBuilder . Reset ( )
stdoutBuilder . Reset ( )
2022-10-23 20:14:45 +05:30
pruneErr := git . NewCommand ( ctx , "remote" , "prune" ) . AddDynamicArguments ( m . GetRemoteName ( ) ) .
2021-10-21 14:15:25 +05:30
SetDescription ( fmt . Sprintf ( "Mirror.runSync %ssPrune references: %s " , wiki , m . Repo . FullName ( ) ) ) .
2022-04-01 08:25:30 +05:30
Run ( & git . RunOpts {
2022-02-11 18:17:22 +05:30
Timeout : timeout ,
Dir : repoPath ,
Stdout : stdoutBuilder ,
Stderr : stderrBuilder ,
} )
2021-10-21 14:15:25 +05:30
if pruneErr != nil {
stdout := stdoutBuilder . String ( )
stderr := stderrBuilder . String ( )
// sanitize the output, since it may contain the remote address, which may
// contain a password
2022-03-31 07:55:40 +05:30
stderrMessage := util . SanitizeCredentialURLs ( stderr )
stdoutMessage := util . SanitizeCredentialURLs ( stdout )
2021-10-21 14:15:25 +05:30
log . Error ( "Failed to prune mirror repository %s%-v references:\nStdout: %s\nStderr: %s\nErr: %v" , wiki , m . Repo , stdoutMessage , stderrMessage , pruneErr )
desc := fmt . Sprintf ( "Failed to prune mirror repository %s'%s' references: %s" , wiki , repoPath , stderrMessage )
2022-10-17 04:59:26 +05:30
if err := system_model . CreateRepositoryNotice ( desc ) ; err != nil {
2021-10-21 14:15:25 +05:30
log . Error ( "CreateRepositoryNotice: %v" , err )
}
// this if will only be reached on a successful prune so try to get the mirror again
}
return pruneErr
}
2021-06-14 22:50:43 +05:30
// runSync returns true if sync finished without error.
2021-12-10 06:57:50 +05:30
func runSync ( ctx context . Context , m * repo_model . Mirror ) ( [ ] * mirrorSyncResult , bool ) {
2021-06-14 22:50:43 +05:30
repoPath := m . Repo . RepoPath ( )
wikiPath := m . Repo . WikiPath ( )
timeout := time . Duration ( setting . Git . Timeout . Mirror ) * time . Second
log . Trace ( "SyncMirrors [repo: %-v]: running git remote update..." , m . Repo )
2022-03-29 22:42:33 +05:30
2022-10-23 20:14:45 +05:30
gitArgs := [ ] git . CmdArg { "remote" , "update" }
2021-06-14 22:50:43 +05:30
if m . EnablePrune {
gitArgs = append ( gitArgs , "--prune" )
}
2022-10-23 20:14:45 +05:30
gitArgs = append ( gitArgs , git . CmdArgCheck ( m . GetRemoteName ( ) ) )
2021-06-14 22:50:43 +05:30
2022-06-11 19:20:14 +05:30
remoteURL , remoteErr := git . GetRemoteURL ( ctx , repoPath , m . GetRemoteName ( ) )
2021-06-14 22:50:43 +05:30
if remoteErr != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: GetRemoteAddress Error %v" , m . Repo , remoteErr )
2022-06-11 19:20:14 +05:30
return nil , false
2021-06-14 22:50:43 +05:30
}
stdoutBuilder := strings . Builder { }
stderrBuilder := strings . Builder { }
2022-02-07 00:31:47 +05:30
if err := git . NewCommand ( ctx , gitArgs ... ) .
2021-06-14 22:50:43 +05:30
SetDescription ( fmt . Sprintf ( "Mirror.runSync: %s" , m . Repo . FullName ( ) ) ) .
2022-04-01 08:25:30 +05:30
Run ( & git . RunOpts {
2022-02-11 18:17:22 +05:30
Timeout : timeout ,
Dir : repoPath ,
Stdout : & stdoutBuilder ,
Stderr : & stderrBuilder ,
} ) ; err != nil {
2021-06-14 22:50:43 +05:30
stdout := stdoutBuilder . String ( )
stderr := stderrBuilder . String ( )
2022-03-31 07:55:40 +05:30
// sanitize the output, since it may contain the remote address, which may contain a password
stderrMessage := util . SanitizeCredentialURLs ( stderr )
stdoutMessage := util . SanitizeCredentialURLs ( stdout )
2021-06-14 22:50:43 +05:30
2021-10-21 14:15:25 +05:30
// Now check if the error is a resolve reference due to broken reference
if strings . Contains ( stderr , "unable to resolve reference" ) && strings . Contains ( stderr , "reference broken" ) {
2022-03-10 15:39:48 +05:30
log . Warn ( "SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune" , m . Repo , stdoutMessage , stderrMessage , err )
2021-10-21 14:15:25 +05:30
err = nil
// Attempt prune
2022-03-31 07:55:40 +05:30
pruneErr := pruneBrokenReferences ( ctx , m , repoPath , timeout , & stdoutBuilder , & stderrBuilder , false )
2021-10-21 14:15:25 +05:30
if pruneErr == nil {
// Successful prune - reattempt mirror
stderrBuilder . Reset ( )
stdoutBuilder . Reset ( )
2022-02-07 00:31:47 +05:30
if err = git . NewCommand ( ctx , gitArgs ... ) .
2021-10-21 14:15:25 +05:30
SetDescription ( fmt . Sprintf ( "Mirror.runSync: %s" , m . Repo . FullName ( ) ) ) .
2022-04-01 08:25:30 +05:30
Run ( & git . RunOpts {
2022-02-11 18:17:22 +05:30
Timeout : timeout ,
Dir : repoPath ,
Stdout : & stdoutBuilder ,
Stderr : & stderrBuilder ,
} ) ; err != nil {
2021-10-21 14:15:25 +05:30
stdout := stdoutBuilder . String ( )
stderr := stderrBuilder . String ( )
// sanitize the output, since it may contain the remote address, which may
// contain a password
2022-03-31 07:55:40 +05:30
stderrMessage = util . SanitizeCredentialURLs ( stderr )
stdoutMessage = util . SanitizeCredentialURLs ( stdout )
2021-10-21 14:15:25 +05:30
}
}
}
// If there is still an error (or there always was an error)
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v" , m . Repo , stdoutMessage , stderrMessage , err )
2021-10-21 14:15:25 +05:30
desc := fmt . Sprintf ( "Failed to update mirror repository '%s': %s" , repoPath , stderrMessage )
2022-10-17 04:59:26 +05:30
if err = system_model . CreateRepositoryNotice ( desc ) ; err != nil {
2021-10-21 14:15:25 +05:30
log . Error ( "CreateRepositoryNotice: %v" , err )
}
return nil , false
2021-06-14 22:50:43 +05:30
}
}
output := stderrBuilder . String ( )
2022-03-29 22:42:33 +05:30
if err := git . WriteCommitGraph ( ctx , repoPath ) ; err != nil {
log . Error ( "SyncMirrors [repo: %-v]: %v" , m . Repo , err )
}
2022-03-30 00:43:41 +05:30
gitRepo , err := git . OpenRepository ( ctx , repoPath )
2021-06-14 22:50:43 +05:30
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: failed to OpenRepository: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
return nil , false
}
log . Trace ( "SyncMirrors [repo: %-v]: syncing releases with tags..." , m . Repo )
if err = repo_module . SyncReleasesWithTags ( m . Repo , gitRepo ) ; err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
}
if m . LFS && setting . LFS . StartServer {
log . Trace ( "SyncMirrors [repo: %-v]: syncing LFS objects..." , m . Repo )
2022-06-11 19:20:14 +05:30
endpoint := lfs . DetermineEndpoint ( remoteURL . String ( ) , m . LFSEndpoint )
2021-11-20 15:04:05 +05:30
lfsClient := lfs . NewClient ( endpoint , nil )
if err = repo_module . StoreMissingLfsObjectsInRepository ( ctx , m . Repo , gitRepo , lfsClient ) ; err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
}
}
gitRepo . Close ( )
log . Trace ( "SyncMirrors [repo: %-v]: updating size of repository" , m . Repo )
2022-06-06 13:31:49 +05:30
if err := repo_module . UpdateRepoSize ( ctx , m . Repo ) ; err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: failed to update size for mirror repository: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
}
if m . Repo . HasWiki ( ) {
log . Trace ( "SyncMirrors [repo: %-v Wiki]: running git remote update..." , m . Repo )
stderrBuilder . Reset ( )
stdoutBuilder . Reset ( )
2022-10-23 20:14:45 +05:30
if err := git . NewCommand ( ctx , "remote" , "update" , "--prune" ) . AddDynamicArguments ( m . GetRemoteName ( ) ) .
2021-06-14 22:50:43 +05:30
SetDescription ( fmt . Sprintf ( "Mirror.runSync Wiki: %s " , m . Repo . FullName ( ) ) ) .
2022-04-01 08:25:30 +05:30
Run ( & git . RunOpts {
2022-02-11 18:17:22 +05:30
Timeout : timeout ,
Dir : wikiPath ,
Stdout : & stdoutBuilder ,
Stderr : & stderrBuilder ,
} ) ; err != nil {
2021-06-14 22:50:43 +05:30
stdout := stdoutBuilder . String ( )
stderr := stderrBuilder . String ( )
2022-03-31 07:55:40 +05:30
// sanitize the output, since it may contain the remote address, which may contain a password
stderrMessage := util . SanitizeCredentialURLs ( stderr )
stdoutMessage := util . SanitizeCredentialURLs ( stdout )
2021-06-14 22:50:43 +05:30
2021-10-21 14:15:25 +05:30
// Now check if the error is a resolve reference due to broken reference
if strings . Contains ( stderrMessage , "unable to resolve reference" ) && strings . Contains ( stderrMessage , "reference broken" ) {
2022-03-10 15:39:48 +05:30
log . Warn ( "SyncMirrors [repo: %-v Wiki]: failed to update mirror wiki repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune" , m . Repo , stdoutMessage , stderrMessage , err )
2021-10-21 14:15:25 +05:30
err = nil
// Attempt prune
2022-03-31 07:55:40 +05:30
pruneErr := pruneBrokenReferences ( ctx , m , repoPath , timeout , & stdoutBuilder , & stderrBuilder , true )
2021-10-21 14:15:25 +05:30
if pruneErr == nil {
// Successful prune - reattempt mirror
stderrBuilder . Reset ( )
stdoutBuilder . Reset ( )
2022-10-23 20:14:45 +05:30
if err = git . NewCommand ( ctx , "remote" , "update" , "--prune" ) . AddDynamicArguments ( m . GetRemoteName ( ) ) .
2021-10-21 14:15:25 +05:30
SetDescription ( fmt . Sprintf ( "Mirror.runSync Wiki: %s " , m . Repo . FullName ( ) ) ) .
2022-04-01 08:25:30 +05:30
Run ( & git . RunOpts {
2022-02-11 18:17:22 +05:30
Timeout : timeout ,
Dir : wikiPath ,
Stdout : & stdoutBuilder ,
Stderr : & stderrBuilder ,
} ) ; err != nil {
2021-10-21 14:15:25 +05:30
stdout := stdoutBuilder . String ( )
stderr := stderrBuilder . String ( )
2022-03-31 07:55:40 +05:30
stderrMessage = util . SanitizeCredentialURLs ( stderr )
stdoutMessage = util . SanitizeCredentialURLs ( stdout )
2021-10-21 14:15:25 +05:30
}
}
}
// If there is still an error (or there always was an error)
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v Wiki]: failed to update mirror repository wiki:\nStdout: %s\nStderr: %s\nErr: %v" , m . Repo , stdoutMessage , stderrMessage , err )
2021-10-21 14:15:25 +05:30
desc := fmt . Sprintf ( "Failed to update mirror repository wiki '%s': %s" , wikiPath , stderrMessage )
2022-10-17 04:59:26 +05:30
if err = system_model . CreateRepositoryNotice ( desc ) ; err != nil {
2021-10-21 14:15:25 +05:30
log . Error ( "CreateRepositoryNotice: %v" , err )
}
return nil , false
2021-06-14 22:50:43 +05:30
}
2022-03-29 22:42:33 +05:30
if err := git . WriteCommitGraph ( ctx , wikiPath ) ; err != nil {
log . Error ( "SyncMirrors [repo: %-v]: %v" , m . Repo , err )
}
2021-06-14 22:50:43 +05:30
}
log . Trace ( "SyncMirrors [repo: %-v Wiki]: git remote update complete" , m . Repo )
}
log . Trace ( "SyncMirrors [repo: %-v]: invalidating mirror branch caches..." , m . Repo )
2022-01-20 04:56:57 +05:30
branches , _ , err := git . GetBranchesByPath ( ctx , m . Repo . RepoPath ( ) , 0 , 0 )
2021-06-14 22:50:43 +05:30
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: failed to GetBranches: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
return nil , false
}
for _ , branch := range branches {
cache . Remove ( m . Repo . GetCommitsCountCacheKey ( branch . Name , true ) )
}
m . UpdatedUnix = timeutil . TimeStampNow ( )
return parseRemoteUpdateOutput ( output ) , true
}
// SyncPullMirror starts the sync of the pull mirror and schedules the next run.
func SyncPullMirror ( ctx context . Context , repoID int64 ) bool {
log . Trace ( "SyncMirrors [repo_id: %v]" , repoID )
defer func ( ) {
err := recover ( )
if err == nil {
return
}
// There was a panic whilst syncMirrors...
2022-03-10 15:39:48 +05:30
log . Error ( "PANIC whilst SyncMirrors[repo_id: %d] Panic: %v\nStacktrace: %s" , repoID , err , log . Stack ( 2 ) )
2021-06-14 22:50:43 +05:30
} ( )
2022-05-20 19:38:52 +05:30
m , err := repo_model . GetMirrorByRepoID ( ctx , repoID )
2021-06-14 22:50:43 +05:30
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v" , repoID , err )
2021-06-14 22:50:43 +05:30
return false
}
2022-05-20 19:38:52 +05:30
_ = m . GetRepository ( ) // force load repository of mirror
2021-06-14 22:50:43 +05:30
2021-12-01 01:36:32 +05:30
ctx , _ , finished := process . GetManager ( ) . AddContext ( ctx , fmt . Sprintf ( "Syncing Mirror %s/%s" , m . Repo . OwnerName , m . Repo . Name ) )
defer finished ( )
2021-06-14 22:50:43 +05:30
log . Trace ( "SyncMirrors [repo: %-v]: Running Sync" , m . Repo )
results , ok := runSync ( ctx , m )
if ! ok {
2022-03-27 20:10:17 +05:30
if err = repo_model . TouchMirror ( ctx , m ) ; err != nil {
log . Error ( "SyncMirrors [repo: %-v]: failed to TouchMirror: %v" , m . Repo , err )
}
2021-06-14 22:50:43 +05:30
return false
}
log . Trace ( "SyncMirrors [repo: %-v]: Scheduling next update" , m . Repo )
m . ScheduleNextUpdate ( )
2022-05-20 19:38:52 +05:30
if err = repo_model . UpdateMirror ( ctx , m ) ; err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: failed to UpdateMirror with next update date: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
return false
}
var gitRepo * git . Repository
if len ( results ) == 0 {
log . Trace ( "SyncMirrors [repo: %-v]: no branches updated" , m . Repo )
} else {
log . Trace ( "SyncMirrors [repo: %-v]: %d branches updated" , m . Repo , len ( results ) )
2022-03-30 00:43:41 +05:30
gitRepo , err = git . OpenRepository ( ctx , m . Repo . RepoPath ( ) )
2021-06-14 22:50:43 +05:30
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: unable to OpenRepository: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
return false
}
defer gitRepo . Close ( )
if ok := checkAndUpdateEmptyRepository ( m , gitRepo , results ) ; ! ok {
return false
}
}
for _ , result := range results {
// Discard GitHub pull requests, i.e. refs/pull/*
2021-12-02 12:58:08 +05:30
if strings . HasPrefix ( result . refName , git . PullPrefix ) {
2021-06-14 22:50:43 +05:30
continue
}
tp , _ := git . SplitRefName ( result . refName )
// Create reference
if result . oldCommitID == gitShortEmptySha {
if tp == git . TagPrefix {
tp = "tag"
} else if tp == git . BranchPrefix {
tp = "branch"
}
commitID , err := gitRepo . GetRefCommitID ( result . refName )
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v" , m . Repo , result . refName , err )
2021-06-14 22:50:43 +05:30
continue
}
2022-11-19 13:42:33 +05:30
notification . NotifySyncPushCommits ( ctx , m . Repo . MustOwner ( ctx ) , m . Repo , & repo_module . PushUpdateOptions {
2021-06-14 22:50:43 +05:30
RefFullName : result . refName ,
OldCommitID : git . EmptySHA ,
NewCommitID : commitID ,
} , repo_module . NewPushCommits ( ) )
2022-11-19 13:42:33 +05:30
notification . NotifySyncCreateRef ( ctx , m . Repo . MustOwner ( ctx ) , m . Repo , tp , result . refName , commitID )
2021-06-14 22:50:43 +05:30
continue
}
// Delete reference
if result . newCommitID == gitShortEmptySha {
2022-11-19 13:42:33 +05:30
notification . NotifySyncDeleteRef ( ctx , m . Repo . MustOwner ( ctx ) , m . Repo , tp , result . refName )
2021-06-14 22:50:43 +05:30
continue
}
// Push commits
2022-01-20 04:56:57 +05:30
oldCommitID , err := git . GetFullCommitID ( gitRepo . Ctx , gitRepo . Path , result . oldCommitID )
2021-06-14 22:50:43 +05:30
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v" , m . Repo , result . oldCommitID , err )
2021-06-14 22:50:43 +05:30
continue
}
2022-01-20 04:56:57 +05:30
newCommitID , err := git . GetFullCommitID ( gitRepo . Ctx , gitRepo . Path , result . newCommitID )
2021-06-14 22:50:43 +05:30
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v" , m . Repo , result . newCommitID , err )
2021-06-14 22:50:43 +05:30
continue
}
commits , err := gitRepo . CommitsBetweenIDs ( newCommitID , oldCommitID )
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: unable to get CommitsBetweenIDs [new_commit_id: %s, old_commit_id: %s]: %v" , m . Repo , newCommitID , oldCommitID , err )
2021-06-14 22:50:43 +05:30
continue
}
2021-08-09 23:38:51 +05:30
theCommits := repo_module . GitToPushCommits ( commits )
2021-06-14 22:50:43 +05:30
if len ( theCommits . Commits ) > setting . UI . FeedMaxCommitNum {
theCommits . Commits = theCommits . Commits [ : setting . UI . FeedMaxCommitNum ]
}
theCommits . CompareURL = m . Repo . ComposeCompareURL ( oldCommitID , newCommitID )
2022-11-19 13:42:33 +05:30
notification . NotifySyncPushCommits ( ctx , m . Repo . MustOwner ( ctx ) , m . Repo , & repo_module . PushUpdateOptions {
2021-06-14 22:50:43 +05:30
RefFullName : result . refName ,
OldCommitID : oldCommitID ,
NewCommitID : newCommitID ,
} , theCommits )
}
log . Trace ( "SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time" , m . Repo )
// Get latest commit date and update to current repository updated time
2022-01-20 04:56:57 +05:30
commitDate , err := git . GetLatestCommitTime ( ctx , m . Repo . RepoPath ( ) )
2021-06-14 22:50:43 +05:30
if err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: unable to GetLatestCommitDate: %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
return false
}
2021-12-12 21:18:20 +05:30
if err = repo_model . UpdateRepositoryUpdatedTime ( m . RepoID , commitDate ) ; err != nil {
2022-03-10 15:39:48 +05:30
log . Error ( "SyncMirrors [repo: %-v]: unable to update repository 'updated_unix': %v" , m . Repo , err )
2021-06-14 22:50:43 +05:30
return false
}
log . Trace ( "SyncMirrors [repo: %-v]: Successfully updated" , m . Repo )
return true
}
2021-12-10 06:57:50 +05:30
func checkAndUpdateEmptyRepository ( m * repo_model . Mirror , gitRepo * git . Repository , results [ ] * mirrorSyncResult ) bool {
2021-06-14 22:50:43 +05:30
if ! m . Repo . IsEmpty {
return true
}
hasDefault := false
hasMaster := false
hasMain := false
defaultBranchName := m . Repo . DefaultBranch
if len ( defaultBranchName ) == 0 {
defaultBranchName = setting . Repository . DefaultBranch
}
firstName := ""
for _ , result := range results {
2021-12-02 12:58:08 +05:30
if strings . HasPrefix ( result . refName , git . PullPrefix ) {
2021-06-14 22:50:43 +05:30
continue
}
tp , name := git . SplitRefName ( result . refName )
if len ( tp ) > 0 && tp != git . BranchPrefix {
continue
}
if len ( firstName ) == 0 {
firstName = name
}
hasDefault = hasDefault || name == defaultBranchName
hasMaster = hasMaster || name == "master"
hasMain = hasMain || name == "main"
}
if len ( firstName ) > 0 {
if hasDefault {
m . Repo . DefaultBranch = defaultBranchName
} else if hasMaster {
m . Repo . DefaultBranch = "master"
} else if hasMain {
m . Repo . DefaultBranch = "main"
} else {
m . Repo . DefaultBranch = firstName
}
// Update the git repository default branch
if err := gitRepo . SetDefaultBranch ( m . Repo . DefaultBranch ) ; err != nil {
if ! git . IsErrUnsupportedVersion ( err ) {
log . Error ( "Failed to update default branch of underlying git repository %-v. Error: %v" , m . Repo , err )
desc := fmt . Sprintf ( "Failed to uupdate default branch of underlying git repository '%s': %v" , m . Repo . RepoPath ( ) , err )
2022-10-17 04:59:26 +05:30
if err = system_model . CreateRepositoryNotice ( desc ) ; err != nil {
2021-06-14 22:50:43 +05:30
log . Error ( "CreateRepositoryNotice: %v" , err )
}
return false
}
}
m . Repo . IsEmpty = false
// Update the is empty and default_branch columns
2022-05-20 19:38:52 +05:30
if err := repo_model . UpdateRepositoryCols ( db . DefaultContext , m . Repo , "default_branch" , "is_empty" ) ; err != nil {
2021-06-14 22:50:43 +05:30
log . Error ( "Failed to update default branch of repository %-v. Error: %v" , m . Repo , err )
desc := fmt . Sprintf ( "Failed to uupdate default branch of repository '%s': %v" , m . Repo . RepoPath ( ) , err )
2022-10-17 04:59:26 +05:30
if err = system_model . CreateRepositoryNotice ( desc ) ; err != nil {
2021-06-14 22:50:43 +05:30
log . Error ( "CreateRepositoryNotice: %v" , err )
}
return false
}
}
return true
}