diff --git a/.gitignore b/.gitignore index 4ee6720..3b03a6a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ _testmain.go *.exe *.test *.prof -vendor/ \ No newline at end of file +vendor/ +.idea/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index df21f17..16f9800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,25 @@ -version 1.2.4 ---- -- Add DOCKER_AGENT_IPADDR env, Register to cluster ipaddr, default 0.0.0.0 auto bind local ipaddr. +# Changelog -version 1.1.2 (not released) ---- -- Change github.com/docker/engine-api(Deprecated) to github.com/docker/docker/client +### version 1.3.4 -version 1.1.1 ---- -- Remove network mode's validate when create container. +- Supported edit single mode and cluster mode containers. -version 1.3.0 ---- -- Add container `--log-opt` property. +### version 1.3.3 -version 1.3.3 ---- - Supported docker compose containers. -version 1.3.4 ---- -- Supported edit single mode and cluster mode containers. +### version 1.3.0 + +- Add container `--log-opt` property. + +### version 1.2.4 + +- Add DOCKER_AGENT_IPADDR env, Register to cluster ipaddr, default 0.0.0.0 auto bind local ipaddr. + +### version 1.1.2 (not released) + +- Change github.com/docker/engine-api(Deprecated) to github.com/docker/docker/client + +### version 1.1.1 + +- Remove network mode's validate when create container. diff --git a/Dockerfile b/Dockerfile index 8be2006..19813f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM frolvlad/alpine-glibc:alpine-3.7 -MAINTAINER bobliu bobliu0909@gmail.com +LABEL author="bobliu" email="bobliu0909@gmail.com" RUN apk add --no-cache bash diff --git a/README.md b/README.md index 68fb25a..5c08f37 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,48 @@ Humpback Agent 主要为 [Humpback](https://github.com/humpback/humpback) 平台提供容器访问功能。[API 文档](https://github.com/humpback/humpback-agent/wiki) +## How to develop + +### #Prepare + +1. First, you need an server that installed and already running docker(recommend Linux). +2. `[Optional]` If you are develop on windows, you should expose docker host with tcp protocol(default is `unix:///var/run/docker.sock`). How to do it? + +```bash +# convert unix-connect as container port 6666, mapping the container port 6666 as host port 1234 +docker run -d --restart=always \ + -p 0.0.0.0:1234:6666 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + alpine/socat \ + tcp-listen:6666,fork,reuseaddr unix-connect:/var/run/docker.sock +``` + +expose your docker host at `tcp::1234` [more info](https://hub.docker.com/r/alpine/socat/). + +### #Init dependencies + +1. Use `go get` install the dependencies. + +**Notice: GOPATH is very important!** + +### #Build & Run + +1. `[Optional]`, If you are develop on windows, change the `conf/app.conf DOCKER_ENDPOINT` config. + +```bash +# For example +DOCKER_ENDPOINT = tcp://192.168.1.200:1234 +# DOCKER_ENDPOINT = unix:///var/run/docker.sock +DOCKER_API_VERSION = v1.21 +``` + +2. cd `humpback-agent` folder, run `go run main.go` +3. `curl http://localhost:8500/v1/dockerinfo` check the service status. + ## License Apache-2.0 + +## Changelog + +[CHANGELOG.md](CHANGELOG.md) diff --git a/conf/app.conf b/conf/app.conf index d2e9ba0..41ad8b4 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -4,6 +4,11 @@ autorender = false copyrequestbody = true servertimeout = 600 +# enalbe validate auth token +ENABLE_AUTHORIZATION = true +AUTHORIZATION_TOKEN = 123456 + +# DOCKER_ENDPOINT = tcp://192.168.1.200:1234 DOCKER_ENDPOINT = unix:///var/run/docker.sock DOCKER_API_VERSION = v1.21 DOCKER_REGISTRY_ADDRESS = http://192.168.2.80 diff --git a/config/config.go b/config/config.go index 1e3155e..4c9c821 100644 --- a/config/config.go +++ b/config/config.go @@ -1,169 +1,161 @@ -package config - -import "github.com/astaxie/beego" -import "github.com/humpback/common/models" -import "github.com/humpback/gounits/network" - -import ( - "fmt" - "net" - "os" - "strconv" - "strings" -) - -var config *models.Config - -// Init - Load config info -func Init() { - envEndpoint := os.Getenv("DOCKER_ENDPOINT") - if envEndpoint == "" { - envEndpoint = beego.AppConfig.DefaultString("DOCKER_ENDPOINT", "unix:///var/run/docker.sock") - } - - envAPIVersion := os.Getenv("DOCKER_API_VERSION") - if envAPIVersion == "" { - envAPIVersion = beego.AppConfig.DefaultString("DOCKER_API_VERSION", "v1.20") - } - - envRegistryAddr := os.Getenv("DOCKER_REGISTRY_ADDRESS") - if envRegistryAddr == "" { - envRegistryAddr = beego.AppConfig.DefaultString("DOCKER_REGISTRY_ADDRESS", "docker.neg") - } - - envNodeHTTPAddr := os.Getenv("DOCKER_NODE_HTTPADDR") - if envNodeHTTPAddr == "" { - envNodeHTTPAddr = beego.AppConfig.DefaultString("DOCKER_NODE_HTTPADDR", "0.0.0.0:8500") - } - - envContainerPortsRange := os.Getenv("DOCKER_CONTAINER_PORTS_RANGE") - if envContainerPortsRange == "" { - envContainerPortsRange = beego.AppConfig.DefaultString("DOCKER_CONTAINER_PORTS_RANGE", "0-0") - } - - var envEnableBuildImg bool - if tempEnableBuildImg := os.Getenv("ENABLE_BUILD_IMAGE"); tempEnableBuildImg != "" { - if tempEnableBuildImg == "1" || tempEnableBuildImg == "true" { - envEnableBuildImg = true - } - } else { - envEnableBuildImg = beego.AppConfig.DefaultBool("ENABLE_BUILD_IMAGE", false) - } - - envComposePath := os.Getenv("DOCKER_COMPOSE_PATH") - if envComposePath == "" { - envComposePath = beego.AppConfig.DefaultString("DOCKER_COMPOSE_PATH", "./compose_files") - } - - var envComposePackageMaxSize int64 - packageMaxSize := os.Getenv("DOCKER_COMPOSE_PACKAGE_MAXSIZE") - if packageMaxSize == "" { - envComposePackageMaxSize = beego.AppConfig.DefaultInt64("DOCKER_COMPOSE_PACKAGE_MAXSIZE", 67108864) - } else { - value, err := strconv.ParseInt(packageMaxSize, 10, 64) - if err != nil { - envComposePackageMaxSize = 67108864 - } else { - envComposePackageMaxSize = value - } - } - - envClusterEnabled := false - enabled := os.Getenv("DOCKER_CLUSTER_ENABLED") - if enabled == "" { - envClusterEnabled = beego.AppConfig.DefaultBool("DOCKER_CLUSTER_ENABLED", false) - } else { - var err error - if envClusterEnabled, err = strconv.ParseBool(enabled); err != nil { - envClusterEnabled = false - } - } - - envClusterURIs := os.Getenv("DOCKER_CLUSTER_URIS") - if envClusterURIs == "" { - envClusterURIs = beego.AppConfig.DefaultString("DOCKER_CLUSTER_URIS", "zk://127.0.0.1:2181") - } - - envClusterName := os.Getenv("DOCKER_CLUSTER_NAME") - if envClusterName == "" { - envClusterName = beego.AppConfig.DefaultString("DOCKER_CLUSTER_NAME", "humpback/center") - } - - envClusterHeartBeat := os.Getenv("DOCKER_CLUSTER_HEARTBEAT") - if envClusterHeartBeat == "" { - envClusterHeartBeat = beego.AppConfig.DefaultString("DOCKER_CLUSTER_HEARTBEAT", "10s") - } - - envClusterTTL := os.Getenv("DOCKER_CLUSTER_TTL") - if envClusterTTL == "" { - envClusterTTL = beego.AppConfig.DefaultString("DOCKER_CLUSTER_TTL", "35s") - } - - var logLevel int - if envLogLevel := os.Getenv("LOG_LEVEL"); envLogLevel != "" { - logLevel, _ = strconv.Atoi(envLogLevel) - } else { - logLevel = beego.AppConfig.DefaultInt("LOG_LEVEL", 3) - } - - config = &models.Config{ - DockerEndPoint: envEndpoint, - DockerAPIVersion: envAPIVersion, - DockerRegistryAddress: envRegistryAddr, - EnableBuildImage: envEnableBuildImg, - DockerComposePath: envComposePath, - DockerComposePackageMaxSize: envComposePackageMaxSize, - DockerNodeHTTPAddr: envNodeHTTPAddr, - DockerContainerPortsRange: envContainerPortsRange, - DockerClusterEnabled: envClusterEnabled, - DockerClusterURIs: envClusterURIs, - DockerClusterName: envClusterName, - DockerClusterHeartBeat: envClusterHeartBeat, - DockerClusterTTL: envClusterTTL, - LogLevel: logLevel, - } -} - -// GetConfig - return config struct -func GetConfig() models.Config { - return *config -} - -// SetVersion - set app version -func SetVersion(version string) { - config.AppVersion = version -} - -// GetNodeHTTPAddrIPPort - return local agent httpaddr info -func GetNodeHTTPAddrIPPort() (string, int, error) { - - httpAddr := strings.TrimSpace(config.DockerNodeHTTPAddr) - if strings.Index(httpAddr, ":") < 0 { - httpAddr = httpAddr + ":" - } - - pAddrStr := strings.SplitN(httpAddr, ":", 2) - if pAddrStr[1] == "" { - httpAddr = httpAddr + "8500" - } - - strHost, strPort, err := net.SplitHostPort(httpAddr) - if err != nil { - return "", 0, err - } - - if strHost == "" || strHost == "0.0.0.0" { - strHost = network.GetDefaultIP() - } - - ip := net.ParseIP(strHost) - if ip == nil { - return "", 0, fmt.Errorf("httpAddr ip invalid") - } - - port, err := strconv.Atoi(strPort) - if err != nil || port > 65535 || port <= 0 { - return "", 0, fmt.Errorf("httpAddr port invalid") - } - return ip.String(), port, nil -} +package config + +import "github.com/astaxie/beego" +import "github.com/humpback/common/models" +import "github.com/humpback/gounits/network" + +import ( + "fmt" + "net" + "os" + "strconv" + "strings" +) + +var config *models.Config +var enableAuthorization bool +var AuthorizationToken string + +/** +Get config from env, if empty, get it from app config. +*/ +func getConfigFromEnvOrAppConfig(key string, defaulValue string) string { + tmpValue := os.Getenv(key) + if tmpValue == "" { + tmpValue = beego.AppConfig.DefaultString(key, defaulValue) + } + return tmpValue +} + +/** +Init - Load config info +*/ +func Init() { + envEndpoint := getConfigFromEnvOrAppConfig("DOCKER_ENDPOINT", "unix:///var/run/docker.sock") + envAPIVersion := getConfigFromEnvOrAppConfig("DOCKER_API_VERSION", "v1.20") + envRegistryAddr := getConfigFromEnvOrAppConfig("DOCKER_REGISTRY_ADDRESS", "docker.neg") + envNodeHTTPAddr := getConfigFromEnvOrAppConfig("DOCKER_NODE_HTTPADDR", "0.0.0.0:8500") + envContainerPortsRange := getConfigFromEnvOrAppConfig("DOCKER_CONTAINER_PORTS_RANGE", "0-0") + envComposePath := getConfigFromEnvOrAppConfig("DOCKER_COMPOSE_PATH", "./compose_files") + AuthorizationToken = getConfigFromEnvOrAppConfig("AUTHORIZATION_TOKEN", "humpback") + + // enable authorization + enableAuthorization = false + if enableAuth := os.Getenv("ENABLE_AUTHORIZATION"); enableAuth != "" { + if enableAuth == "1" || enableAuth == "true" { + enableAuthorization = true + } + } else { + enableAuthorization = beego.AppConfig.DefaultBool("ENABLE_AUTHORIZATION", false) + } + var envEnableBuildImg bool + if tempEnableBuildImg := os.Getenv("ENABLE_BUILD_IMAGE"); tempEnableBuildImg != "" { + if tempEnableBuildImg == "1" || tempEnableBuildImg == "true" { + envEnableBuildImg = true + } + } else { + envEnableBuildImg = beego.AppConfig.DefaultBool("ENABLE_BUILD_IMAGE", false) + } + + var envComposePackageMaxSize int64 + packageMaxSize := os.Getenv("DOCKER_COMPOSE_PACKAGE_MAXSIZE") + if packageMaxSize == "" { + envComposePackageMaxSize = beego.AppConfig.DefaultInt64("DOCKER_COMPOSE_PACKAGE_MAXSIZE", 67108864) + } else { + value, err := strconv.ParseInt(packageMaxSize, 10, 64) + if err != nil { + envComposePackageMaxSize = 67108864 + } else { + envComposePackageMaxSize = value + } + } + + envClusterEnabled := false + enabled := os.Getenv("DOCKER_CLUSTER_ENABLED") + if enabled == "" { + envClusterEnabled = beego.AppConfig.DefaultBool("DOCKER_CLUSTER_ENABLED", false) + } else { + var err error + if envClusterEnabled, err = strconv.ParseBool(enabled); err != nil { + envClusterEnabled = false + } + } + + // Cluster config + envClusterURIs := getConfigFromEnvOrAppConfig("DOCKER_CLUSTER_URIS", "zk://127.0.0.1:2181") + envClusterName := getConfigFromEnvOrAppConfig("DOCKER_CLUSTER_NAME", "humpback/center") + envClusterHeartBeat := getConfigFromEnvOrAppConfig("DOCKER_CLUSTER_HEARTBEAT", "10s") + envClusterTTL := getConfigFromEnvOrAppConfig("DOCKER_CLUSTER_TTL", "35s") + + var logLevel int + if envLogLevel := os.Getenv("LOG_LEVEL"); envLogLevel != "" { + logLevel, _ = strconv.Atoi(envLogLevel) + } else { + logLevel = beego.AppConfig.DefaultInt("LOG_LEVEL", 3) + } + + config = &models.Config{ + DockerEndPoint: envEndpoint, + DockerAPIVersion: envAPIVersion, + DockerRegistryAddress: envRegistryAddr, + EnableBuildImage: envEnableBuildImg, + DockerComposePath: envComposePath, + DockerComposePackageMaxSize: envComposePackageMaxSize, + DockerNodeHTTPAddr: envNodeHTTPAddr, + DockerContainerPortsRange: envContainerPortsRange, + DockerClusterEnabled: envClusterEnabled, + DockerClusterURIs: envClusterURIs, + DockerClusterName: envClusterName, + DockerClusterHeartBeat: envClusterHeartBeat, + DockerClusterTTL: envClusterTTL, + LogLevel: logLevel, + } +} + +// GetConfig - return config struct +func GetConfig() models.Config { + return *config +} + +func GetEnableAuthorization() bool { + return enableAuthorization +} + +// SetVersion - set app version +func SetVersion(version string) { + config.AppVersion = version +} + +// GetNodeHTTPAddrIPPort - return local agent httpaddr info +func GetNodeHTTPAddrIPPort() (string, int, error) { + + httpAddr := strings.TrimSpace(config.DockerNodeHTTPAddr) + if strings.Index(httpAddr, ":") < 0 { + httpAddr = httpAddr + ":" + } + + pAddrStr := strings.SplitN(httpAddr, ":", 2) + if pAddrStr[1] == "" { + httpAddr = httpAddr + "8500" + } + + strHost, strPort, err := net.SplitHostPort(httpAddr) + if err != nil { + return "", 0, err + } + + if strHost == "" || strHost == "0.0.0.0" { + strHost = network.GetDefaultIP() + } + + ip := net.ParseIP(strHost) + if ip == nil { + return "", 0, fmt.Errorf("httpAddr ip invalid") + } + + port, err := strconv.Atoi(strPort) + if err != nil || port > 65535 || port <= 0 { + return "", 0, fmt.Errorf("httpAddr port invalid") + } + return ip.String(), port, nil +} diff --git a/controllers/base.go b/controllers/base.go index e3898e7..856fc88 100644 --- a/controllers/base.go +++ b/controllers/base.go @@ -1,7 +1,7 @@ package controllers import "github.com/humpback/common/models" -import "github.com/humpback/humpback-agent/config" +import "humpback-agent/config" import "github.com/astaxie/beego" import "github.com/docker/docker/client" diff --git a/controllers/container.go b/controllers/container.go index 293fd2d..99f491e 100644 --- a/controllers/container.go +++ b/controllers/container.go @@ -1,14 +1,13 @@ package controllers import "github.com/humpback/common/models" -import "github.com/humpback/humpback-agent/config" +import "humpback-agent/config" import gonetwork "github.com/humpback/gounits/network" import "github.com/astaxie/beego" import "github.com/docker/docker/api/types" import "github.com/docker/docker/api/types/container" import "github.com/docker/docker/api/types/network" import "github.com/docker/go-connections/nat" -import "golang.org/x/net/context" import ( "encoding/json" @@ -17,6 +16,7 @@ import ( "strconv" "strings" "time" + "context" ) // ContainerController - handle http request for container diff --git a/controllers/faq.go b/controllers/faq.go index c5281e5..77e4b57 100644 --- a/controllers/faq.go +++ b/controllers/faq.go @@ -1,6 +1,6 @@ package controllers -import "github.com/humpback/humpback-agent/config" +import "humpback-agent/config" // FaqController - faq info type FaqController struct { diff --git a/controllers/image.go b/controllers/image.go index 20ef43d..a9373b5 100644 --- a/controllers/image.go +++ b/controllers/image.go @@ -3,7 +3,6 @@ package controllers import "github.com/docker/docker/api/types" import "github.com/docker/docker/client" import "github.com/humpback/common/models" -import "golang.org/x/net/context" import ( "encoding/json" @@ -11,6 +10,7 @@ import ( "io" "strings" "time" + "context" ) // ImageController - handle http request for image diff --git a/controllers/info.go b/controllers/info.go index b8a946c..568ef65 100644 --- a/controllers/info.go +++ b/controllers/info.go @@ -1,6 +1,8 @@ package controllers -import "context" +import ( + "context" +) type InfoController struct { baseController diff --git a/ErrorCode.md b/docs/ErrorCode.md similarity index 100% rename from ErrorCode.md rename to docs/ErrorCode.md diff --git a/filters/authorization_filter.go b/filters/authorization_filter.go new file mode 100644 index 0000000..124ccd6 --- /dev/null +++ b/filters/authorization_filter.go @@ -0,0 +1,37 @@ +package filters + +import ( + "encoding/json" + + "humpback-agent/config" + + "github.com/astaxie/beego/context" +) + +func Authorization(ctx *context.Context) { + var errs []string + var authHeaders = ctx.Request.Header["Authorization"] + var statusCode = 400 + var token = "" + + if len(authHeaders) > 0 { + token = authHeaders[0] + if token != config.AuthorizationToken { + errs = append(errs, "Invalid authorization token.") + statusCode = 403 + } + } else { + errs = append(errs, "Please provide authorization token in header.") + } + + if len(errs) > 0 { + errData := map[string]interface{}{ + "Code": statusCode, + "Detail": errs, + } + ctx.ResponseWriter.Header().Set("Content-Type", "application/json") + ctx.ResponseWriter.WriteHeader(statusCode) + body, _ := json.Marshal(errData) + ctx.ResponseWriter.Write(body) + } +} diff --git a/glide.lock b/glide.lock deleted file mode 100644 index 6d9043f..0000000 --- a/glide.lock +++ /dev/null @@ -1,57 +0,0 @@ -hash: 1eebe7390a6ff9ca843037e2c699cb453bb4448b44922cec0e072cbe77706ee7 -updated: 2016-12-20T13:47:09.6042193+08:00 -imports: -- name: github.com/astaxie/beego - version: 90999717dda6193c6f970eec52e289b0a9e74998 - subpackages: - - config - - context - - grace - - logs - - plugins/cors - - session - - toolbox - - utils -- name: github.com/docker/distribution - version: 2d500932f2c7f032d2abc51f2ef940871d541319 - subpackages: - - digest - - reference -- name: github.com/docker/docker - version: 88862e707affccf9c46347361689b2ffef39dc5c - subpackages: - - api/types - - api/types/blkiodev - - api/types/container - - api/types/events - - api/types/filters - - api/types/mount - - api/types/network - - api/types/reference - - api/types/registry - - api/types/strslice - - api/types/swarm - - api/types/time - - api/types/versions - - api/types/volume - - client - - pkg/tlsconfig -- name: github.com/docker/go-connections - version: 990a1a1a70b0da4c4cb70e117971a4f0babfbf1a - subpackages: - - nat - - sockets - - tlsconfig -- name: github.com/docker/go-units - version: e30f1e79f3cd72542f2026ceec18d3bd67ab859c -- name: github.com/Microsoft/go-winio - version: 24a3e3d3fc7451805e09d11e11e95d9a0a4f205e -- name: github.com/opencontainers/runc - version: 9a1e53eafc0ebef4712b442a0aa53a1a7016a23d - subpackages: - - libcontainer/user -- name: github.com/pkg/errors - version: 248dadf4e9068a0b3e79f02ed0a610d935de5302 -- name: github.com/Sirupsen/logrus - version: 881bee4e20a5d11a6a88a5667c6f292072ac1963 -testImports: [] diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 5d16d83..0000000 --- a/glide.yaml +++ /dev/null @@ -1,12 +0,0 @@ -package: humpback-agent -ignore: -- golang.org/x -import: -- package: github.com/astaxie/beego - subpackages: - - context - - plugins/cors -- package: github.com/docker/docker - version: ^1.12.5 - subpackages: - - client diff --git a/main.go b/main.go index 1c04225..520f7cb 100644 --- a/main.go +++ b/main.go @@ -1,67 +1,69 @@ -package main - -import "github.com/astaxie/beego" -import "github.com/astaxie/beego/plugins/cors" -import "github.com/humpback/common/models" -import "github.com/humpback/humpback-agent/config" -import "github.com/humpback/humpback-agent/controllers" -import "github.com/humpback/humpback-agent/routers" -import "github.com/humpback/humpback-center/cluster/types" - -import ( - "os" - "os/signal" - "syscall" -) - -func main() { - - config.Init() - config.SetVersion("1.3.7") - - controllers.Init() - var conf = config.GetConfig() - beego.BConfig.MaxMemory = conf.DockerComposePackageMaxSize - composeStorage, err := models.NewComposeStorage(conf.DockerComposePath) - if err != nil { - beego.Error("compose storage error, " + err.Error()) - return - } - routers.Init(composeStorage) - - beego.InsertFilter("*", beego.BeforeRouter, cors.Allow(&cors.Options{ - AllowOrigins: []string{"*"}, - AllowMethods: []string{"GET", "POST", "PUT", "DELETE"}, - AllowHeaders: []string{"Origin", "Accept", "Content-Type", "Authorization"}, - AllowCredentials: true, - })) - beego.SetLogFuncCall(true) - beego.SetLevel(conf.LogLevel) - - ipAddr, bindPort, err := config.GetNodeHTTPAddrIPPort() - if err != nil { - beego.Error("agent httpaddr error:" + err.Error()) - } - - beego.BConfig.Listen.HTTPPort = bindPort - if conf.DockerClusterEnabled { - clusterOptions := types.NewNodeRegisterOptions(ipAddr, bindPort, &conf) - if err := types.NodeRegister(clusterOptions); err != nil { - beego.Error("cluster node register error:" + err.Error()) - return - } - } - - go signalListen() - beego.Run() -} - -func signalListen() { - c := make(chan os.Signal) - signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) - for { - <-c - types.NodeClose() - os.Exit(0) - } -} +package main + +import "github.com/astaxie/beego" +import "github.com/astaxie/beego/plugins/cors" +import "github.com/humpback/common/models" +import "humpback-agent/config" +import "humpback-agent/controllers" +import "humpback-agent/routers" +import "github.com/humpback/humpback-center/cluster/types" + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +func main() { + + config.Init() + config.SetVersion("1.3.7") + + controllers.Init() + var conf = config.GetConfig() + beego.BConfig.MaxMemory = conf.DockerComposePackageMaxSize + composeStorage, err := models.NewComposeStorage(conf.DockerComposePath) + if err != nil { + beego.Error("compose storage error, " + err.Error()) + return + } + routers.Init(composeStorage) + + beego.InsertFilter("*", beego.BeforeRouter, cors.Allow(&cors.Options{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{"GET", "POST", "PUT", "DELETE"}, + AllowHeaders: []string{"Origin", "Accept", "Content-Type", "Authorization"}, + AllowCredentials: true, + })) + beego.SetLogFuncCall(true) + beego.SetLevel(conf.LogLevel) + + ipAddr, bindPort, err := config.GetNodeHTTPAddrIPPort() + if err != nil { + beego.Error("agent httpaddr error:" + err.Error()) + } + + beego.BConfig.Listen.HTTPPort = bindPort + if conf.DockerClusterEnabled { + clusterOptions := types.NewNodeRegisterOptions(ipAddr, bindPort, &conf) + if err := types.NodeRegister(clusterOptions); err != nil { + beego.Error("cluster node register error:" + err.Error()) + return + } + } + + go signalListen() + fmt.Println("Server started...") + beego.Run() +} + +func signalListen() { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) + for { + <-c + types.NodeClose() + os.Exit(0) + } +} diff --git a/models/compose_storage.go b/models/compose_storage.go new file mode 100644 index 0000000..df6f795 --- /dev/null +++ b/models/compose_storage.go @@ -0,0 +1,374 @@ +package models + +import "github.com/humpback/gounits/compress/tarlib" +import "github.com/humpback/gounits/httpx" +import "github.com/humpback/gounits/system" +import "github.com/humpback/gounits/utils" +import yaml "gopkg.in/yaml.v2" + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ( + //ErrProjectNameInvalid is exported + ErrProjectNameInvalid = errors.New("project name invalid") + //ErrProjectPackageFileInvalid is exported + ErrProjectPackageFileInvalid = errors.New("project package file invalid") +) + +//ProjectJSON is exported +type ProjectJSON struct { + Name string `json:"Name"` + HashCode string `json:"HashCode"` + PackageFile string `json:"PackageFile"` + Timestamp int64 `json:"Timestamp"` +} + +//ProjectData is exported +type ProjectData struct { + *ProjectJSON + ComposeFile string `json:"ComposeFile"` + ComposeBytes []byte `json:"ComposeBytes"` +} + +func validateProjectName(projectName string) error { + + if strings.TrimSpace(projectName) == "" { + return ErrProjectNameInvalid + } + return nil +} + +func validateComposeData(composeData string) error { + + var err error + buffer := []byte(composeData) + if len(buffer) == 0 { + return fmt.Errorf("compose data format invalid, data is empty") + } + template := map[string]interface{}{} + if err = yaml.Unmarshal(buffer, &template); err != nil { + return fmt.Errorf("compose data format invalid, %s", err) + } + return nil +} + +func projectComposeFilePath(rootPath string, projectName string, hashCode string) string { + + projectDir := filepath.Dir(projectFilePath(rootPath, projectName)) + composeFilePath := filepath.Clean(projectDir + "/" + hashCode + "/docker-compose.yaml") + return composeFilePath +} + +func projectFilePath(rootPath string, projectName string) string { + + projectFilePath := filepath.Clean(rootPath + "/" + projectName + "/project.json") + return projectFilePath +} + +func projectHashCode(projectName string, packageFile string, timeStamp int64) string { + + projectStr := fmt.Sprintf("%s-%s-%d", projectName, packageFile, timeStamp) + sum := sha256.Sum256([]byte(projectStr)) + return fmt.Sprintf("%x", sum) +} + +func savePackageFile(rootPath string, projectName string, packageFile string, packageFileBytes []byte) (string, error) { + + timeStamp := time.Now().Unix() + fps := strings.Split(packageFile, ".") + sum := sha256.Sum256([]byte(fmt.Sprintf("%s%s%d", projectName, fps[0], timeStamp))) + fps[0] = fps[0] + "-" + fmt.Sprintf("%x", sum)[:6] + packageFile = strings.Join(fps, ".") + + packageFilePath := filepath.Clean(rootPath + "/_uploads/" + projectName + "/" + packageFile) + packageDir := filepath.Dir(packageFilePath) + if err := system.MakeDirectory(packageDir); err != nil { + return "", err + } + + f, err := os.OpenFile(packageFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return "", err + } + + defer f.Close() + //io.Copy(f, reader) + if _, err := f.Write(packageFileBytes); err != nil { + return "", err + } + return packageFile, nil +} + +func removePackageFile(rootPath string, projectName string, packageFile string, removeAll bool) error { + + if !removeAll { + if _, err := url.ParseRequestURI(packageFile); err == nil { + packageFile = filepath.Base(packageFile) + } + } + + packageFileDir := filepath.Clean(rootPath + "/_uploads/" + projectName) + if !removeAll { + packageFilePath := filepath.Clean(packageFileDir + "/" + packageFile) + return os.Remove(packageFilePath) + } + return os.RemoveAll(packageFileDir) +} + +func extractPackageFile(rootPath string, projectName string, packageFile string, hashCode string) error { + + packageFilePath := filepath.Clean(rootPath + "/_uploads/" + projectName + "/" + packageFile) + if _, err := url.ParseRequestURI(packageFile); err == nil { + err = httpx.GetFile(context.Background(), packageFilePath, packageFile, nil, nil) + if err != nil { + return fmt.Errorf("project %s package %s download failure.", projectName, packageFile) + } + } + + if !system.FileExist(packageFilePath) { + return fmt.Errorf("project %s package %s not exists.", projectName, packageFile) + } + + composeFilePath := projectComposeFilePath(rootPath, projectName, hashCode) + composeFileDir := filepath.Dir(composeFilePath) + err := tarlib.TarAutoDeCompress(packageFilePath, composeFileDir) + if err != nil { + return fmt.Errorf("project %s package %s extract failure.", projectName, packageFile) + } + return nil +} + +//ComposeStorage is exported +//compose project local storage. +type ComposeStorage struct { + sync.RWMutex + rootPath string + filterDirs []string +} + +//NewComposeStorage is exported +func NewComposeStorage(rootPath string) (*ComposeStorage, error) { + + var err error + rootPath, err = filepath.Abs(rootPath) + if err != nil { + return nil, err + } + + if err = system.MakeDirectory(rootPath + "/_uploads"); err != nil { + return nil, err + } + + return &ComposeStorage{ + rootPath: rootPath, + filterDirs: []string{"_uploads"}, + }, nil +} + +//ProjectJSON is exported +func (storage *ComposeStorage) ProjectJSON(projectName string) (*ProjectJSON, error) { + + if err := validateProjectName(projectName); err != nil { + return nil, err + } + + storage.Lock() + defer storage.Unlock() + projectFilePath := projectFilePath(storage.rootPath, projectName) + data, err := ioutil.ReadFile(projectFilePath) + if err != nil { + if os.IsNotExist(err) || strings.Index(err.Error(), "no such file or directory") > 0 { + return nil, os.ErrNotExist + } + return nil, err + } + + projectJSON := &ProjectJSON{} + err = json.Unmarshal(data, projectJSON) + if err != nil { + return nil, err + } + return projectJSON, nil +} + +//SetProjectJSON is exported +func (storage *ComposeStorage) SetProjectJSON(projectJSON *ProjectJSON) error { + + if err := validateProjectName(projectJSON.Name); err != nil { + return err + } + + storage.Lock() + defer storage.Unlock() + data, err := json.Marshal(projectJSON) + if err != nil { + return err + } + + projectFilePath := projectFilePath(storage.rootPath, projectJSON.Name) + projectDir := filepath.Dir(projectFilePath) + if err = system.MakeDirectory(projectDir); err != nil { + return err + } + return ioutil.WriteFile(projectFilePath, data, 0777) +} + +//ProjectSpecs is exported +func (storage *ComposeStorage) ProjectSpecs() ([]*ProjectData, error) { + + fis, err := ioutil.ReadDir(storage.rootPath) + if err != nil { + return nil, err + } + + projectDataArray := []*ProjectData{} + for _, fi := range fis { + if fi.IsDir() && !utils.Contains(fi.Name(), storage.filterDirs) { + projectData, _ := storage.ProjectSpec(fi.Name()) + if projectData != nil { + projectDataArray = append(projectDataArray, projectData) + } + } + } + return projectDataArray, nil +} + +//ProjectSpec is exported +func (storage *ComposeStorage) ProjectSpec(projectName string) (*ProjectData, error) { + + projectJSON, err := storage.ProjectJSON(projectName) + if err != nil { + return nil, err + } + + storage.Lock() + defer storage.Unlock() + composeFilePath := projectComposeFilePath(storage.rootPath, projectName, projectJSON.HashCode) + composeBytes, err := ioutil.ReadFile(composeFilePath) + if err != nil { + return nil, err + } + + err = validateComposeData(string(composeBytes)) + if err != nil { + return nil, err + } + + return &ProjectData{ + ProjectJSON: projectJSON, + ComposeFile: composeFilePath, + ComposeBytes: composeBytes, + }, nil +} + +//CreateProjectSpec is exported +func (storage *ComposeStorage) CreateProjectSpec(createProject CreateProject) (*ProjectData, error) { + + var ( + err error + projectJSON *ProjectJSON + ) + + if err = validateProjectName(createProject.Name); err != nil { + return nil, err + } + + if err = validateComposeData(createProject.ComposeData); err != nil { + return nil, err + } + + projectJSON, err = storage.ProjectJSON(createProject.Name) + if projectJSON != nil || !os.IsNotExist(err) { + return nil, os.ErrExist + } + + timeStamp := time.Now().Unix() + hashCode := projectHashCode(createProject.Name, createProject.PackageFile, timeStamp) + projectJSON = &ProjectJSON{ + Name: createProject.Name, + HashCode: hashCode, + PackageFile: createProject.PackageFile, + Timestamp: timeStamp, + } + + if err = storage.SetProjectJSON(projectJSON); err != nil { + return nil, err + } + + storage.Lock() + defer func() { + storage.Unlock() + if err != nil { + storage.RemoveProjectSpec(projectJSON.Name) + } + }() + + composeFilePath := projectComposeFilePath(storage.rootPath, projectJSON.Name, hashCode) + composeFileDir := filepath.Dir(composeFilePath) + err = system.MakeDirectory(composeFileDir) + if err != nil { + return nil, err + } + + composeBytes := []byte(createProject.ComposeData) + err = ioutil.WriteFile(composeFilePath, composeBytes, 0777) + if err != nil { + return nil, err + } + + if projectJSON.PackageFile != "" { + err = extractPackageFile(storage.rootPath, projectJSON.Name, projectJSON.PackageFile, projectJSON.HashCode) + if err != nil { + return nil, err + } + } + + return &ProjectData{ + ProjectJSON: projectJSON, + ComposeFile: composeFilePath, + ComposeBytes: composeBytes, + }, nil +} + +//RemoveProjectSpec is exported +func (storage *ComposeStorage) RemoveProjectSpec(projectName string) error { + + projectJSON, err := storage.ProjectJSON(projectName) + if err != nil { + return err + } + + storage.Lock() + defer storage.Unlock() + if projectJSON.PackageFile != "" { + removePackageFile(storage.rootPath, projectJSON.Name, projectJSON.PackageFile, false) + } + projectDir := filepath.Dir(projectFilePath(storage.rootPath, projectJSON.Name)) + return os.RemoveAll(projectDir) +} + +//SaveProjectPackageFile is exported +func (storage *ComposeStorage) SaveProjectPackageFile(projectName string, packageFile string, packageFileBytes []byte) (string, error) { + + if err := validateProjectName(projectName); err != nil { + return "", err + } + + if packageFile == "" { + return "", ErrProjectPackageFileInvalid + } + return savePackageFile(storage.rootPath, projectName, packageFile, packageFileBytes) +} diff --git a/models/config.go b/models/config.go new file mode 100644 index 0000000..250a642 --- /dev/null +++ b/models/config.go @@ -0,0 +1,20 @@ +package models + +// Config - config info +type Config struct { + AppVersion string `json:"AppVersion"` + DockerEndPoint string `json:"DockerEndPoint"` + DockerAPIVersion string `json:"DockerAPIVersion"` + DockerRegistryAddress string `json:"-"` + EnableBuildImage bool `json:"-"` + DockerComposePath string `json:"DockerComposePath"` + DockerComposePackageMaxSize int64 `json:"DockerComposePackageMaxSize"` + DockerNodeHTTPAddr string `json:"DockerNodeHTTPAddr"` + DockerContainerPortsRange string `json:"DockerContainerPortsRange"` + DockerClusterEnabled bool `json:"DockerClusterEnabled"` + DockerClusterURIs string `json:"DockerClusterURIs"` + DockerClusterName string `json:"DockerClusterName"` + DockerClusterHeartBeat string `json:"DockerClusterHeartBeat"` + DockerClusterTTL string `json:"DockerClusterTTL"` + LogLevel int `json:"LogLevel"` +} diff --git a/models/container.go b/models/container.go new file mode 100644 index 0000000..fa84a51 --- /dev/null +++ b/models/container.go @@ -0,0 +1,226 @@ +package models + +import "github.com/docker/docker/api/types" +import "github.com/docker/docker/api/types/container" +import units "github.com/docker/go-units" + +import ( + "strconv" + "strings" + "time" +) + +var ( + RestartPolicyType = map[string]interface{}{ + "no": true, + "always": true, + "on-failure": true, + } + + ActionType = map[string]interface{}{ + "start": true, + "stop": true, + "restart": true, + "kill": true, + "pause": true, + "unpause": true, + "rename": true, + "upgrade": true, + } +) + +// Container - define container info struct +type Container struct { + ID string `json:"Id"` + Image string `json:"Image"` + Command string `json:"Command"` + CommandWithoutEntryPoint string `json:"CommandWithoutEntryPoint"` + Name string `json:"Name"` + Ports []PortBinding `json:"Ports"` + Volumes []VolumesBinding `json:"Volumes"` + DNS []string `json:"Dns"` + Env []string `json:"Env"` + HostName string `json:"HostName"` + NetworkMode string `json:"NetworkMode"` + Status interface{} `json:"Status,omitempty"` + RestartPolicy string `json:"RestartPolicy,omitempty"` + RestartRetryCount int `json:"RestartRetryCount,omitempty"` + ExtraHosts []string `json:"Extrahosts"` + CPUShares int64 `json:"CPUShares,omitempty"` + Memory int64 `json:"Memory,omitempty"` + SHMSize int64 `json:"SHMSize,omitempty"` + Links []string `json:"Links"` + Ulimits []*units.Ulimit `json:"Ulimits"` + Labels map[string]string `json:"Labels"` + LogConfig container.LogConfig `json:"LogConfig"` +} + +// PortBinding - define container port binding info struct +type PortBinding struct { + PrivatePort int `json:"PrivatePort"` + PublicPort int `json:"PublicPort"` + Type string `json:"Type"` + IP string `json:"Ip"` +} + +// VolumesBinding - define container volumes binding info struct +type VolumesBinding struct { + ContainerVolume string `json:"ContainerVolume"` + HostVolume string `json:"HostVolume"` + BindMount string `json:"BindMount,omitempty"` +} + +// ContainerOperate - define container http request struct +type ContainerOperate struct { + Action string `json:"Action"` + Container string `json:"Container"` + ImageTag string `json:"ImageTag"` + NewName string `json:"NewName"` +} + +// ContainerLog - define container logs info struct +type ContainerLog struct { + Stdout []string `json:"StdOut"` + Stderr []string `json:"StdErr"` +} + +// ContainerStatsFromDocker - define container stats info from dockerapi +type ContainerStatsFromDocker struct { + Network struct { + RxBytes float64 `json:"rx_bytes"` + TxBytes float64 `json:"tx_bytes"` + } `json:"network"` + + PreCPUStats struct { + CPUUsage struct { + TotalUsage float64 `json:"total_usage"` + } `json:"cpu_usage"` + SystemCPUUsage float64 `json:"system_cpu_usage"` + } `json:"precpu_stats"` + + CPUStats struct { + CPUUsage struct { + TotalUsage float64 `json:"total_usage"` + } `json:"cpu_usage"` + SystemCPUUsage float64 `json:"system_cpu_usage"` + } `json:"cpu_stats"` + + MemoryStats struct { + Usage float64 `json:"usage"` + Limit float64 `json:"limit"` + } `json:"memory_stats"` + + BlkioStats struct { + IOServiceBytesRecursive []IORecursive `json:"io_service_bytes_recursive"` + } `json:"blkio_stats"` +} + +// IORecursive - define container stats's IORecursive +type IORecursive struct { + Major int64 `json:"major"` + Minor int64 `json:"minor"` + Op string `json:"op"` + Value int64 `json:"value"` +} + +// ContainerStats - define container stats info struct +type ContainerStats struct { + ContainerName string `json:"ContainerName"` + ContainerID string `json:"ContainerID"` + CPUUsage string `json:"CPUUsage"` + MemoryUsage int64 `json:"MemoryUsage"` + MemoryLimit int64 `json:"MemoryLimit"` + MemoryPercent float64 `json:"MemoryPercent"` + NetworkIn int64 `json:"NetworkIn"` + NetworkOut int64 `json:"NetowrkOut"` + IOBytesRead int64 `json:"IOBytesRead"` + IOBytesWrite int64 `json:"IOBytesWrite"` +} + +// ContainerStatsWithError - define container stats info struct +type ContainerStatsWithError struct { + Stats ContainerStats `json:"ContainerStats"` + Error interface{} `json:"Error"` +} + +// Parse - parse container from original container info +func (container *Container) Parse(origContainer *types.ContainerJSON) { + container.ID = origContainer.ID + container.Name = strings.Replace(origContainer.Name, "/", "", 1) + container.Image = origContainer.Config.Image + container.Env = origContainer.Config.Env + container.DNS = origContainer.HostConfig.DNS + container.HostName = origContainer.Config.Hostname + container.NetworkMode = origContainer.HostConfig.NetworkMode.NetworkName() + container.Status = origContainer.State + container.Labels = origContainer.Config.Labels + + command := origContainer.Path + " " + strings.Join(origContainer.Args, " ") + container.Command = strings.TrimLeft(command, " ") + container.CommandWithoutEntryPoint = strings.Join(origContainer.Config.Cmd, " ") + + for item := range origContainer.Config.ExposedPorts { + containerPort, _ := strconv.Atoi(item.Port()) + tempPortbinding := PortBinding{ + PrivatePort: containerPort, + PublicPort: 0, + Type: item.Proto(), + IP: "0.0.0.0", + } + hostportBind := origContainer.NetworkSettings.Ports[item] + if hostportBind == nil { + hostportBind = origContainer.HostConfig.PortBindings[item] + } + if hostportBind != nil && len(hostportBind) > 0 { + hostPort, _ := strconv.Atoi(hostportBind[0].HostPort) + tempPortbinding.PublicPort = hostPort + if hostportBind[0].HostIP != "" { + tempPortbinding.IP = hostportBind[0].HostIP + } + } + + container.Ports = append(container.Ports, tempPortbinding) + } + + for _, mount := range origContainer.Mounts { + tempVolumeBinding := VolumesBinding{ + ContainerVolume: mount.Destination, + HostVolume: mount.Source, + } + container.Volumes = append(container.Volumes, tempVolumeBinding) + } + + container.RestartPolicy = origContainer.HostConfig.RestartPolicy.Name + container.RestartRetryCount = origContainer.HostConfig.RestartPolicy.MaximumRetryCount + container.ExtraHosts = origContainer.HostConfig.ExtraHosts + container.CPUShares = origContainer.HostConfig.CPUShares + container.Memory = origContainer.HostConfig.Memory / 1024 / 1024 + container.SHMSize = origContainer.HostConfig.ShmSize + container.Links = origContainer.HostConfig.Links + container.Ulimits = origContainer.HostConfig.Ulimits + container.LogConfig = origContainer.HostConfig.LogConfig +} + +// ContainerStateString - convert container state to short string format +func ContainerStateString(state *types.ContainerState) string { + + startedAt, _ := time.Parse(time.RFC3339Nano, state.StartedAt) + if state.Running { + if state.Paused { + return "Paused" + } + if state.Restarting { + return "Restarting" + } + return "Running" + } + + if state.Dead { + return "Dead" + } + + if startedAt.IsZero() { + return "Created" + } + return "Exited" +} diff --git a/models/error.go b/models/error.go new file mode 100644 index 0000000..86a3cf0 --- /dev/null +++ b/models/error.go @@ -0,0 +1,14 @@ +package models + +// ErrorMap - error map msg +var ErrorMap = map[int]string{ + 20001: "Container already created, but cannot start", + 20002: "Get container info failed when upgrade container", + 20003: "Rename container failed when upgrade or update container", + 20004: "Try pull image failed when upgrade container", + 20005: "Stop container failed when upgrade or update container", + 20006: "Cannot create new container when upgrade container", + 20007: "Cannot start new container when upgrade container", + 20008: "Cannot delete old container when upgrade container, but image is upgrade succeed", + 21001: "Try pull image failed when create or update container", +} diff --git a/models/image.go b/models/image.go new file mode 100644 index 0000000..bc52099 --- /dev/null +++ b/models/image.go @@ -0,0 +1,12 @@ +package models + +// Image - define image info struct +type Image struct { + Image string `json:'Image'` +} + +// ImageTags - define image tags info struct +type ImageTags struct { + ImageName string `json:"ImageName"` + Tags []string `json:"Tags"` +} diff --git a/models/project.go b/models/project.go new file mode 100644 index 0000000..04f8cc3 --- /dev/null +++ b/models/project.go @@ -0,0 +1,55 @@ +package models + +import "github.com/docker/libcompose/project" + +var ( + ProjectActions = map[string]interface{}{ + "start": true, + "stop": true, + "restart": true, + "kill": true, + "pause": true, + "unpause": true, + } +) + +// CreateProject - define compose create project struct +type CreateProject struct { + Name string `json:"Name"` + ComposeData string `json:"ComposeData"` + PackageFile string `json:"PackageFile"` +} + +// OperateProject - define compose operate project struct +type OperateProject struct { + Name string `json:"Name"` + Action string `json:"Action"` +} + +// ProjectBase - define project base info struct +type ProjectBase struct { + Name string `json:"Name"` + HashCode string `json:"HashCode"` + Timestamp int64 `json:"Timestamp"` + Containers project.InfoSet `json:"Containers"` +} + +// ProjectInfo - define compose project info struct +type ProjectInfo struct { + ProjectBase + ComposeData string `json:"ComposeData"` + PackageFile string `json:"PackageFile"` +} + +// ProjectStatus - define compose project status struct +type ProjectStatus struct { + Name string `json:"Name"` + Status interface{} `json:"Status"` +} + +// ProjectUploadStatus - define compose project package file upload struct +type ProjectUploadStatus struct { + Name string `json:"Name"` + PackageFile string `json:"PackageFile"` + Status interface{} `json:"Status"` +} diff --git a/models/service.go b/models/service.go new file mode 100644 index 0000000..05da3d1 --- /dev/null +++ b/models/service.go @@ -0,0 +1,9 @@ +package models + +import "github.com/docker/docker/api/types" + +//ServiceState - define compose service state struct +type ServiceState struct { + Name string `json:"Name"` + *types.ContainerState +} diff --git a/routers/router.go b/routers/router.go index 467bfd8..69cfc86 100644 --- a/routers/router.go +++ b/routers/router.go @@ -2,8 +2,12 @@ package routers import "github.com/astaxie/beego" import "github.com/humpback/common/models" -import "github.com/humpback/humpback-agent/controllers" -import "github.com/humpback/humpback-agent/validators" +import ( + "humpback-agent/config" + "humpback-agent/controllers" + "humpback-agent/filters" + "humpback-agent/validators" +) // Init - Init routers func Init(composeStorage *models.ComposeStorage) { @@ -49,6 +53,9 @@ func Init(composeStorage *models.ComposeStorage) { serviceRouters, ) beego.AddNamespace(ns, agentSpace) + if config.GetEnableAuthorization() { + beego.InsertFilter("/*", beego.BeforeExec, filters.Authorization) + } beego.InsertFilter("/dockerapi/v2/containers", beego.BeforeExec, validators.CreateContainerValidator) beego.InsertFilter("/v1/containers", beego.BeforeExec, validators.CreateContainerValidator)