mirror of https://github.com/go-gitea/gitea.git
#835: Realtime webhooks
parent
2b1442f3df
commit
fa298a2c30
|
@ -16,6 +16,7 @@ import (
|
||||||
"github.com/codegangsta/cli"
|
"github.com/codegangsta/cli"
|
||||||
|
|
||||||
"github.com/gogits/gogs/models"
|
"github.com/gogits/gogs/models"
|
||||||
|
"github.com/gogits/gogs/modules/httplib"
|
||||||
"github.com/gogits/gogs/modules/log"
|
"github.com/gogits/gogs/modules/log"
|
||||||
"github.com/gogits/gogs/modules/setting"
|
"github.com/gogits/gogs/modules/setting"
|
||||||
"github.com/gogits/gogs/modules/uuid"
|
"github.com/gogits/gogs/modules/uuid"
|
||||||
|
@ -193,6 +194,12 @@ func runServ(c *cli.Context) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Send deliver hook request.
|
||||||
|
resp, err := httplib.Head(setting.AppUrl + setting.AppSubUrl + repoUserName + "/" + repoName + "/hooks/trigger").Response()
|
||||||
|
if err == nil {
|
||||||
|
resp.Body.Close()
|
||||||
|
}
|
||||||
|
|
||||||
// Update key activity.
|
// Update key activity.
|
||||||
key, err := models.GetPublicKeyById(keyId)
|
key, err := models.GetPublicKeyById(keyId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -451,6 +451,7 @@ func runWeb(ctx *cli.Context) {
|
||||||
m.Get("/archive/*", repo.Download)
|
m.Get("/archive/*", repo.Download)
|
||||||
m.Get("/pulls2/", repo.PullRequest2)
|
m.Get("/pulls2/", repo.PullRequest2)
|
||||||
m.Get("/milestone2/", repo.Milestones2)
|
m.Get("/milestone2/", repo.Milestones2)
|
||||||
|
m.Head("/hooks/trigger", repo.TriggerHook)
|
||||||
|
|
||||||
m.Group("", func() {
|
m.Group("", func() {
|
||||||
m.Get("/src/*", repo.Home)
|
m.Get("/src/*", repo.Home)
|
||||||
|
|
|
@ -91,8 +91,8 @@ ENABLE_REVERSE_PROXY_AUTO_REGISTRATION = false
|
||||||
DISABLE_MINIMUM_KEY_SIZE_CHECK = false
|
DISABLE_MINIMUM_KEY_SIZE_CHECK = false
|
||||||
|
|
||||||
[webhook]
|
[webhook]
|
||||||
; Cron task interval in minutes
|
; Hook task queue length
|
||||||
TASK_INTERVAL = 1
|
QUEUE_LENGTH = 1000
|
||||||
; Deliver timeout in seconds
|
; Deliver timeout in seconds
|
||||||
DELIVER_TIMEOUT = 5
|
DELIVER_TIMEOUT = 5
|
||||||
; Allow insecure certification
|
; Allow insecure certification
|
||||||
|
|
2
gogs.go
2
gogs.go
|
@ -17,7 +17,7 @@ import (
|
||||||
"github.com/gogits/gogs/modules/setting"
|
"github.com/gogits/gogs/modules/setting"
|
||||||
)
|
)
|
||||||
|
|
||||||
const APP_VER = "0.6.2.0725 Beta"
|
const APP_VER = "0.6.3.0725 Beta"
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||||
|
|
|
@ -431,6 +431,8 @@ func CommitRepoAction(userId, repoUserId int64, userName, actEmail string,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = CreateHookTask(&HookTask{
|
if err = CreateHookTask(&HookTask{
|
||||||
|
RepoID: repo.Id,
|
||||||
|
HookID: w.Id,
|
||||||
Type: w.HookTaskType,
|
Type: w.HookTaskType,
|
||||||
Url: w.Url,
|
Url: w.Url,
|
||||||
BasePayload: payload,
|
BasePayload: payload,
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/gogits/gogs/modules/httplib"
|
"github.com/gogits/gogs/modules/httplib"
|
||||||
|
@ -259,7 +260,9 @@ func (p Payload) GetJSONPayload() ([]byte, error) {
|
||||||
|
|
||||||
// HookTask represents a hook task.
|
// HookTask represents a hook task.
|
||||||
type HookTask struct {
|
type HookTask struct {
|
||||||
Id int64
|
ID int64 `xorm:"pk autoincr"`
|
||||||
|
RepoID int64 `xorm:"INDEX"`
|
||||||
|
HookID int64
|
||||||
Uuid string
|
Uuid string
|
||||||
Type HookTaskType
|
Type HookTaskType
|
||||||
Url string
|
Url string
|
||||||
|
@ -269,6 +272,7 @@ type HookTask struct {
|
||||||
EventType HookEventType
|
EventType HookEventType
|
||||||
IsSsl bool
|
IsSsl bool
|
||||||
IsDelivered bool
|
IsDelivered bool
|
||||||
|
Delivered int64
|
||||||
IsSucceed bool
|
IsSucceed bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -287,87 +291,137 @@ func CreateHookTask(t *HookTask) error {
|
||||||
|
|
||||||
// UpdateHookTask updates information of hook task.
|
// UpdateHookTask updates information of hook task.
|
||||||
func UpdateHookTask(t *HookTask) error {
|
func UpdateHookTask(t *HookTask) error {
|
||||||
_, err := x.Id(t.Id).AllCols().Update(t)
|
_, err := x.Id(t.ID).AllCols().Update(t)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
type hookQueue struct {
|
||||||
// Prevent duplicate deliveries.
|
// Make sure one repository only occur once in the queue.
|
||||||
// This happens with massive hook tasks cannot finish delivering
|
lock sync.Mutex
|
||||||
// before next shooting starts.
|
repoIDs map[int64]bool
|
||||||
isShooting = false
|
|
||||||
)
|
|
||||||
|
|
||||||
// DeliverHooks checks and delivers undelivered hooks.
|
queue chan int64
|
||||||
// FIXME: maybe can use goroutine to shoot a number of them at same time?
|
}
|
||||||
func DeliverHooks() {
|
|
||||||
if isShooting {
|
func (q *hookQueue) removeRepoID(id int64) {
|
||||||
|
q.lock.Lock()
|
||||||
|
defer q.lock.Unlock()
|
||||||
|
delete(q.repoIDs, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *hookQueue) addRepoID(id int64) {
|
||||||
|
q.lock.Lock()
|
||||||
|
if q.repoIDs[id] {
|
||||||
|
q.lock.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
isShooting = true
|
q.repoIDs[id] = true
|
||||||
defer func() { isShooting = false }()
|
q.lock.Unlock()
|
||||||
|
q.queue <- id
|
||||||
|
}
|
||||||
|
|
||||||
tasks := make([]*HookTask, 0, 10)
|
// AddRepoID adds repository ID to hook delivery queue.
|
||||||
|
func (q *hookQueue) AddRepoID(id int64) {
|
||||||
|
go q.addRepoID(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
var HookQueue *hookQueue
|
||||||
|
|
||||||
|
func deliverHook(t *HookTask) {
|
||||||
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
|
timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
|
||||||
x.Where("is_delivered=?", false).Iterate(new(HookTask),
|
req := httplib.Post(t.Url).SetTimeout(timeout, timeout).
|
||||||
func(idx int, bean interface{}) error {
|
Header("X-Gogs-Delivery", t.Uuid).
|
||||||
t := bean.(*HookTask)
|
Header("X-Gogs-Event", string(t.EventType)).
|
||||||
req := httplib.Post(t.Url).SetTimeout(timeout, timeout).
|
SetTLSClientConfig(&tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify})
|
||||||
Header("X-Gogs-Delivery", t.Uuid).
|
|
||||||
Header("X-Gogs-Event", string(t.EventType)).
|
|
||||||
SetTLSClientConfig(&tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify})
|
|
||||||
|
|
||||||
switch t.ContentType {
|
switch t.ContentType {
|
||||||
case JSON:
|
case JSON:
|
||||||
req = req.Header("Content-Type", "application/json").Body(t.PayloadContent)
|
req = req.Header("Content-Type", "application/json").Body(t.PayloadContent)
|
||||||
case FORM:
|
case FORM:
|
||||||
req.Param("payload", t.PayloadContent)
|
req.Param("payload", t.PayloadContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.IsDelivered = true
|
||||||
|
|
||||||
|
// FIXME: record response.
|
||||||
|
switch t.Type {
|
||||||
|
case GOGS:
|
||||||
|
{
|
||||||
|
if resp, err := req.Response(); err != nil {
|
||||||
|
log.Error(5, "Delivery: %v", err)
|
||||||
|
} else {
|
||||||
|
resp.Body.Close()
|
||||||
|
t.IsSucceed = true
|
||||||
}
|
}
|
||||||
|
}
|
||||||
t.IsDelivered = true
|
case SLACK:
|
||||||
|
{
|
||||||
// FIXME: record response.
|
if resp, err := req.Response(); err != nil {
|
||||||
switch t.Type {
|
log.Error(5, "Delivery: %v", err)
|
||||||
case GOGS:
|
} else {
|
||||||
{
|
defer resp.Body.Close()
|
||||||
if _, err := req.Response(); err != nil {
|
contents, err := ioutil.ReadAll(resp.Body)
|
||||||
log.Error(5, "Delivery: %v", err)
|
if err != nil {
|
||||||
|
log.Error(5, "%s", err)
|
||||||
|
} else {
|
||||||
|
if string(contents) != "ok" {
|
||||||
|
log.Error(5, "slack failed with: %s", string(contents))
|
||||||
} else {
|
} else {
|
||||||
t.IsSucceed = true
|
t.IsSucceed = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
case SLACK:
|
|
||||||
{
|
|
||||||
if res, err := req.Response(); err != nil {
|
|
||||||
log.Error(5, "Delivery: %v", err)
|
|
||||||
} else {
|
|
||||||
defer res.Body.Close()
|
|
||||||
contents, err := ioutil.ReadAll(res.Body)
|
|
||||||
if err != nil {
|
|
||||||
log.Error(5, "%s", err)
|
|
||||||
} else {
|
|
||||||
if string(contents) != "ok" {
|
|
||||||
log.Error(5, "slack failed with: %s", string(contents))
|
|
||||||
} else {
|
|
||||||
t.IsSucceed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Delivered = time.Now().UTC().UnixNano()
|
||||||
|
if t.IsSucceed {
|
||||||
|
log.Trace("Hook delivered(%s): %s", t.Uuid, t.PayloadContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeliverHooks checks and delivers undelivered hooks.
|
||||||
|
func DeliverHooks() {
|
||||||
|
tasks := make([]*HookTask, 0, 10)
|
||||||
|
x.Where("is_delivered=?", false).Iterate(new(HookTask),
|
||||||
|
func(idx int, bean interface{}) error {
|
||||||
|
t := bean.(*HookTask)
|
||||||
|
deliverHook(t)
|
||||||
tasks = append(tasks, t)
|
tasks = append(tasks, t)
|
||||||
|
|
||||||
if t.IsSucceed {
|
|
||||||
log.Trace("Hook delivered(%s): %s", t.Uuid, t.PayloadContent)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
// Update hook task status.
|
// Update hook task status.
|
||||||
for _, t := range tasks {
|
for _, t := range tasks {
|
||||||
if err := UpdateHookTask(t); err != nil {
|
if err := UpdateHookTask(t); err != nil {
|
||||||
log.Error(4, "UpdateHookTask(%d): %v", t.Id, err)
|
log.Error(4, "UpdateHookTask(%d): %v", t.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
HookQueue = &hookQueue{
|
||||||
|
lock: sync.Mutex{},
|
||||||
|
repoIDs: make(map[int64]bool),
|
||||||
|
queue: make(chan int64, setting.Webhook.QueueLength),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start listening on new hook requests.
|
||||||
|
for repoID := range HookQueue.queue {
|
||||||
|
HookQueue.removeRepoID(repoID)
|
||||||
|
|
||||||
|
tasks = make([]*HookTask, 0, 5)
|
||||||
|
if err := x.Where("repo_id=? AND is_delivered=?", repoID, false).Find(&tasks); err != nil {
|
||||||
|
log.Error(4, "Get repository(%d) hook tasks: %v", repoID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, t := range tasks {
|
||||||
|
deliverHook(t)
|
||||||
|
if err := UpdateHookTask(t); err != nil {
|
||||||
|
log.Error(4, "UpdateHookTask(%d): %v", t.ID, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func InitDeliverHooks() {
|
||||||
|
go DeliverHooks()
|
||||||
|
}
|
||||||
|
|
File diff suppressed because one or more lines are too long
|
@ -15,7 +15,6 @@ var c = New()
|
||||||
|
|
||||||
func NewCronContext() {
|
func NewCronContext() {
|
||||||
c.AddFunc("Update mirrors", "@every 1h", models.MirrorUpdate)
|
c.AddFunc("Update mirrors", "@every 1h", models.MirrorUpdate)
|
||||||
c.AddFunc("Deliver hooks", fmt.Sprintf("@every %dm", setting.Webhook.TaskInterval), models.DeliverHooks)
|
|
||||||
if setting.Git.Fsck.Enable {
|
if setting.Git.Fsck.Enable {
|
||||||
c.AddFunc("Repository health check", fmt.Sprintf("@every %dh", setting.Git.Fsck.Interval), models.GitFsck)
|
c.AddFunc("Repository health check", fmt.Sprintf("@every %dh", setting.Git.Fsck.Interval), models.GitFsck)
|
||||||
}
|
}
|
||||||
|
|
|
@ -76,7 +76,7 @@ var (
|
||||||
|
|
||||||
// Webhook settings.
|
// Webhook settings.
|
||||||
Webhook struct {
|
Webhook struct {
|
||||||
TaskInterval int
|
QueueLength int
|
||||||
DeliverTimeout int
|
DeliverTimeout int
|
||||||
SkipTLSVerify bool
|
SkipTLSVerify bool
|
||||||
}
|
}
|
||||||
|
@ -555,7 +555,7 @@ func newNotifyMailService() {
|
||||||
|
|
||||||
func newWebhookService() {
|
func newWebhookService() {
|
||||||
sec := Cfg.Section("webhook")
|
sec := Cfg.Section("webhook")
|
||||||
Webhook.TaskInterval = sec.Key("TASK_INTERVAL").MustInt(1)
|
Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
|
||||||
Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5)
|
Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5)
|
||||||
Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool()
|
Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool()
|
||||||
}
|
}
|
||||||
|
|
|
@ -68,6 +68,7 @@ func GlobalInit() {
|
||||||
|
|
||||||
models.HasEngine = true
|
models.HasEngine = true
|
||||||
cron.NewCronContext()
|
cron.NewCronContext()
|
||||||
|
models.InitDeliverHooks()
|
||||||
log.NewGitLogger(path.Join(setting.LogRootPath, "http.log"))
|
log.NewGitLogger(path.Join(setting.LogRootPath, "http.log"))
|
||||||
}
|
}
|
||||||
if models.EnableSQLite3 {
|
if models.EnableSQLite3 {
|
||||||
|
|
|
@ -190,7 +190,10 @@ func Http(ctx *middleware.Context) {
|
||||||
refName := fields[2]
|
refName := fields[2]
|
||||||
|
|
||||||
// FIXME: handle error.
|
// FIXME: handle error.
|
||||||
models.Update(refName, oldCommitId, newCommitId, authUsername, username, reponame, authUser.Id)
|
if err = models.Update(refName, oldCommitId, newCommitId, authUsername, username, reponame, authUser.Id); err == nil {
|
||||||
|
models.HookQueue.AddRepoID(repo.Id)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
lastLine = lastLine + size
|
lastLine = lastLine + size
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -634,3 +634,7 @@ func GitHooksEditPost(ctx *middleware.Context) {
|
||||||
}
|
}
|
||||||
ctx.Redirect(ctx.Repo.RepoLink + "/settings/hooks/git")
|
ctx.Redirect(ctx.Repo.RepoLink + "/settings/hooks/git")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TriggerHook(ctx *middleware.Context) {
|
||||||
|
models.HookQueue.AddRepoID(ctx.Repo.Repository.Id)
|
||||||
|
}
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
0.6.2.0725 Beta
|
0.6.3.0725 Beta
|
Loading…
Reference in New Issue