Merge pull request 'fix: webfinger before loading search results for /explore/users/' (!7) from task-600 into federation

Reviewed-on: #7

CODEOWNERS

@@ -1,32 +0,0 @@
-# This file describes the expected reviewers for a PR based on the changed
-# files. Unlike what the name of the file suggests they don't own the code, but
-# merely have a good understanding of that area of the codebase and therefore
-# are usually suited as a reviewer.
-
-
-# Please mind the alphabetic order of reviewers.
-
-# Files related to the CI of the Forgejo project.
-.forgejo/.* @dachary @earl-warren
-
-# Files related to frontend development.
-
-# Javascript and CSS code.
-web_src/.* @caesar @crystal @gusted
-
-# HTML templates used by the backend.
-templates/.* @caesar @crystal @gusted
-
-# Files related to Go development.
-
-# The modules usually don't require much knowledge about Forgejo and could
-# be reviewed by Go developers.
-modules/.* @dachary @earl-warren @gusted
-
-# Models has code related to SQL queries, general database knowledge and XORM.
-models/.* @dachary @earl-warren @gusted
-
-# The routers directory contains the most amount code that requires a good grasp
-# of how Forgejo comes together. It's tedious to write good integration testing
-# for code that lives in here.
-routers/.* @dachary @earl-warren @gusted

models/federation/federation.go

@@ -3,10 +3,13 @@ package federation
 import (
 	"context"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"xorm.io/builder"
 )
 
 // HookTask represents a hook task.
@@ -51,23 +54,70 @@ type FederatedUser struct {
 	FederationHostID int64  `xorm:"INDEX"`
 }
 
-func CreateFederatedUser(ctx context.Context, u *user.User, host *FederatedHost) error {
+func CreateFederatedUser(ctx context.Context, alias string, website string, hostname string) (*user.User, error) {
 	engine := db.GetEngine(ctx)
-	//	_, err := engine.
+
-	//		Insert(u)
+	// create FederatedHost
-	//	if err != nil {
+	exists, err := FederatedHostExists(ctx, hostname)
-	//		return err
+	if err != nil {
-	//	}
+		return nil, err
+	}
+
+	var federatedHost FederatedHost
+	if exists {
+		x, err := GetFederatdHost(ctx, hostname)
+		federatedHost = *x
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		federatedHost := new(FederatedHost)
+		federatedHost.HostFqdn = hostname
+		if err = federatedHost.Save(ctx); err != nil {
+			return nil, err
+		}
+	}
+
+	// create user.User
+	u := new(user.User)
+	u.Name = "@" + alias + "@" + hostname
+	//panic(u.Name)
+	u.Email = alias + "@" + hostname
+	u.Website = website
+	u.KeepEmailPrivate = true
+
+	exist, err := user.GetUser(ctx, u)
+	if err != nil {
+		return nil, err
+	}
+	if exist {
+		return u, nil // TODO: must also check for federatedUser existence
+	}
+
+	if err = createUser(ctx, u); err != nil {
+		return nil, err
+	}
 
 	federatedUser := new(FederatedUser)
 	federatedUser.ExternalID = u.Name
 	federatedUser.UserID = u.ID
-	federatedUser.FederationHostID = host.ID
+	federatedUser.FederationHostID = federatedHost.ID
-	_, err := engine.Insert(federatedUser)
+	exist, err = engine.Get(federatedUser)
-	return err
+	if err != nil {
+		return nil, err
+	}
+
+	if !exist {
+		_, err = engine.Insert(federatedUser)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return u, nil
 }
 
-func CreatUser(ctx context.Context, u *user.User) error {
+func createUser(ctx context.Context, u *user.User) error {
 	// set system defaults
 	u.Visibility = setting.Service.DefaultUserVisibilityMode
 	u.AllowCreateOrganization = setting.Service.DefaultAllowCreateOrganization && !setting.Admin.DisableRegularOrgCreation
@@ -161,3 +211,49 @@ func CreatUser(ctx context.Context, u *user.User) error {
 	return committer.Commit()
 
 }
+
+func GetRemoteUsersWithNoLocalFollowers(ctx context.Context, olderThan time.Duration, page int) ([]user.User, error) {
+	limit := 40
+	offset := page * limit
+	var users []user.User
+
+	err := db.GetEngine(ctx).
+		Table("user").
+		Where("num_followers = 0").
+		And(builder.Lt{"user.created_unix": time.Now().Add(-olderThan).Unix()}).
+		Join("inner", "federated_user", "federated_user.user_id = user.id").
+		Limit(limit, offset).
+		Find(&users)
+
+	if err != nil {
+		log.Trace("Error: GetRemoteUserWithNoLocalFollowers: %w", err)
+		return nil, err
+	}
+	return users, nil
+
+}
+
+func GetRemotePersons(ctx context.Context, page int) ([]FederatedUser, error) {
+	limit := 1
+	offset := page * limit
+	var federatedUsers []FederatedUser
+
+	err := db.GetEngine(ctx).
+		Table("federated_user").
+		Limit(limit, offset).
+		Find(&federatedUsers)
+
+		//	TODO: this doesn't work, so fetching federated_user and then getting user. How to make this work?
+		//	var users []user.User
+		//	err := db.GetEngine(ctx).
+		//		Table("user").
+		//		Join("inner", "federated_user", "federated_user.user_id = user.id").
+		//		Limit(limit, offset).
+		//		Find(&users)
+
+	if err != nil {
+		log.Trace("Error: GetRemotePersons: %w", err)
+		return nil, err
+	}
+	return federatedUsers, nil
+}

options/locale/locale_en-US.ini

@@ -2921,6 +2921,8 @@ dashboard.start_schedule_tasks = Start schedule tasks
 dashboard.sync_branch.started = Branches Sync started
 dashboard.sync_tag.started = Tags Sync started
 dashboard.rebuild_issue_indexer = Rebuild issue indexer
+dashboard.remote_actor_cleanup = Clean remote actors with no local followers
+dashboard.remote_actor_update = Update remote actors' data
 
 users.user_manage_panel = Manage user accounts
 users.new_account = Create User Account

routers/web/explore/user.go

@@ -93,13 +93,6 @@ func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions,
 
 	opts.Keyword = ctx.FormTrim("q")
 	opts.OrderBy = orderBy
-	if len(opts.Keyword) == 0 || isKeywordValid(opts.Keyword) {
-		users, count, err = user_model.SearchUsers(ctx, opts)
-		if err != nil {
-			ctx.ServerError("SearchUsers", err)
-			return
-		}
-	}
 
 	if len(opts.Keyword) > 0 && forgefed.IsFingerable(opts.Keyword) {
 		webfingerRes, err := forgefed.WebFingerLookup(opts.Keyword)
@@ -125,6 +118,14 @@ func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions,
 		//		}
 	}
 
+	if len(opts.Keyword) == 0 || isKeywordValid(opts.Keyword) {
+		users, count, err = user_model.SearchUsers(ctx, opts)
+		if err != nil {
+			ctx.ServerError("SearchUsers", err)
+			return
+		}
+	}
+
 	if isSitemap {
 		m := sitemap.NewSitemap()
 		for _, item := range users {

services/cron/tasks_basic.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/services/actions"
 	"code.gitea.io/gitea/services/auth"
+	forgefed_service "code.gitea.io/gitea/services/forgefed"
 	"code.gitea.io/gitea/services/migrations"
 	mirror_service "code.gitea.io/gitea/services/mirror"
 	packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup"
@@ -187,7 +188,40 @@ func initBasicTasks() {
 	if setting.Packages.Enabled {
 		registerCleanupPackages()
 	}
+
 	if setting.Actions.Enabled {
 		registerActionsCleanup()
 	}
+
+	if setting.Federation.Enabled {
+		registerCleanupRemotePersonsWithNoFollowers()
+		registerUpdateRemotePersons()
+	}
+
+}
+
+func registerCleanupRemotePersonsWithNoFollowers() {
+	RegisterTaskFatal("remote_actor_cleanup", &OlderThanConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    true,
+			RunAtStart: true,
+			Schedule:   "@midnight",
+		},
+		OlderThan: 24 * time.Hour,
+	}, func(ctx context.Context, _ *user_model.User, config Config) error {
+		acConfig := config.(*OlderThanConfig)
+		return forgefed_service.CleanUpRemotePersons(ctx, acConfig.OlderThan)
+	})
+}
+
+func registerUpdateRemotePersons() {
+	RegisterTaskFatal("remote_actor_update", &OlderThanConfig{
+		BaseConfig: BaseConfig{
+			Enabled:    true,
+			RunAtStart: true,
+			Schedule:   "@every 6h",
+		},
+	}, func(ctx context.Context, _ *user_model.User, config Config) error {
+		return forgefed_service.UpdatePersonActor(ctx)
+	})
 }

services/forgefed/actor.go

@@ -6,9 +6,12 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 
 	"code.gitea.io/gitea/models/federation"
 	"code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	user_service "code.gitea.io/gitea/services/user"
 
 	ap "github.com/go-ap/activitypub"
 )
@@ -40,57 +43,136 @@ func GetActor(id string) (*ap.Actor, error) {
 	return actorObj, nil
 }
 
+func GetPersonAvatar(ctx context.Context, person *ap.Person) ([]byte, error) {
+
+	avatarObj := new(ap.Image)
+	ap.CopyItemProperties(avatarObj, person.Icon)
+	log.Info("Getting avatar from link : %s", avatarObj.URL.GetLink().String())
+
+	r, err := http.Get(avatarObj.URL.GetLink().String())
+	if err != nil {
+		log.Error("Got error while fetching avatar fn: %w", err)
+		return nil, err
+	}
+	defer r.Body.Close()
+	return io.ReadAll(r.Body)
+}
+
 func SavePerson(ctx context.Context, person *ap.Person) (*user.User, error) {
 
 	fmt.Println(person.ID.String())
 	hostname, err := GetHostnameFromResource(person.ID.String())
 
-	exists, err := federation.FederatedHostExists(ctx, hostname)
+	u, err := federation.CreateFederatedUser(ctx, person.PreferredUsername.String(), person.URL.GetID().String(), hostname)
 	if err != nil {
+		log.Error("Got error while saving person: %w", err)
 		return nil, err
 	}
 
-	var federatedHost federation.FederatedHost
+	avatar, err := GetPersonAvatar(ctx, person)
-	if exists {
-		x, err := federation.GetFederatdHost(ctx, hostname)
-		federatedHost = *x
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		federatedHost := new(federation.FederatedHost)
-		federatedHost.HostFqdn = hostname
-		if err = federatedHost.Save(ctx); err != nil {
-			return nil, err
-		}
-	}
-
 	if err != nil {
+		log.Error("Got error while fetching avatar: %w", err)
 		return nil, err
 	}
 
-	u := new(user.User)
+	if u.IsUploadAvatarChanged(avatar) {
-	u.Name = "@" + person.PreferredUsername.String() + "@" + hostname
+		_ = user_service.UploadAvatar(ctx, u, avatar)
-	//panic(u.Name)
-	u.Email = person.PreferredUsername.String() + "@" + hostname
-	u.Website = person.URL.GetID().String()
-	u.KeepEmailPrivate = true
-
-	exist, err := user.GetUser(ctx, u)
-	if err != nil {
-		return nil, err
-	}
-	if exist {
-		return u, nil // TODO: must also check for federatedUser existence
-	}
-
-	if err = federation.CreatUser(ctx, u); err != nil {
-		return nil, err
-	}
-
-	if err = federation.CreateFederatedUser(ctx, u, &federatedHost); err != nil {
-		return nil, err
 	}
 
 	return u, nil
 }
+
+func GetActorFromUser(ctx context.Context, u *user.User) (*ap.Actor, error) {
+
+	alias := u.Name
+
+	webfingerRes, err := WebFingerLookup(alias)
+	if err != nil {
+		return nil, err
+	}
+
+	actorID := webfingerRes.GetActorLink().Href
+
+	return GetActor(actorID)
+}
+
+// Clean up remote actors (persons) without any followers in local instance
+func CleanUpRemotePersons(ctx context.Context, olderThan time.Duration) error {
+	page := 0
+	for {
+		users, err := federation.GetRemoteUsersWithNoLocalFollowers(ctx, olderThan, page)
+		if len(users) == 0 {
+			break
+		}
+		if err != nil {
+			log.Trace("Error: CleanUpRemotePersons: %v", err)
+			return err
+		}
+
+		for _, u := range users {
+			err = user_service.DeleteUser(ctx, &u, false)
+			if err != nil {
+				log.Trace("Error: CleanUpRemotePersons: %v", err)
+				return err
+			}
+		}
+		page += 1
+	}
+	return nil
+}
+
+func UpdatePersonActor(ctx context.Context) error {
+	// NOTE: change of any of these don't matter at this point since we are
+	// ignoring actor's PreferredUsername and using their address to generate
+	// username and email. Ask suggestions from other devs.
+	//
+	//	fmt.Println(person.ID.String())
+	//	hostname, err := GetHostnameFromResource(person.ID.String())
+	//
+	//
+	//	u := new(user.User)
+	//	u.Name = "@" + person.PreferredUsername.String() + "@" + hostname
+	//	//panic(u.Name)
+	//	u.Email = person.PreferredUsername.String() + "@" + hostname
+	//	u.Website = person.URL.GetID().String()
+	//	u.KeepEmailPrivate = true
+
+	page := 0
+	for {
+		federatedUsers, err := federation.GetRemotePersons(ctx, page)
+		if len(federatedUsers) == 0 {
+			break
+		}
+		if err != nil {
+			log.Trace("Error: UpdatePersonActor: %v", err)
+			return err
+		}
+
+		for _, f := range federatedUsers {
+			log.Info("Updating users, got %s", f.ExternalID)
+			u, err := user.GetUserByName(ctx, f.ExternalID)
+			if err != nil {
+				log.Error("Got error while getting user: %w", err)
+				return err
+			}
+
+			person, err := GetActorFromUser(ctx, u)
+			if err != nil {
+				log.Error("Got error while fetching actor: %w", err)
+				return err
+			}
+
+			avatar, err := GetPersonAvatar(ctx, person)
+			if err != nil {
+				log.Error("Got error while fetching avatar: %w", err)
+				return err
+			}
+
+			if u.IsUploadAvatarChanged(avatar) {
+				_ = user_service.UploadAvatar(ctx, u, avatar)
+			}
+		}
+		page += 1
+	}
+	return nil
+}