diff --git a/debian/rake-tasks.sh b/debian/rake-tasks.sh
index 8369be166c..4492bc6d8c 100755
--- a/debian/rake-tasks.sh
+++ b/debian/rake-tasks.sh
@@ -20,7 +20,8 @@ if ! [ -f "${gitlab_app_root}/config/secrets.yml" ]; then
 fi
 
 # Remove config files (already removed by upstream)
-rm -rf /etc/gitlab/initializers/devise_async.rb
+dpkg-maintscript-helper rm_conffile \
+  /etc/gitlab/initializers/devise_async.rb 8.8.2+dfsg-5 gitlab -- "$@"
 
 # Check if the db is already present
 if [ "$(LANG=C su postgres -c "psql gitlab_production -c \"\d\"")" = \