From 4b0c0ad288617398fcbabbdefed09de7d3d5e74f Mon Sep 17 00:00:00 2001
From: naiba
Date: Mon, 21 Jun 2021 21:30:42 +0800
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20=E4=BC=98=E5=8C=96=E5=BF=BD?=
=?UTF-8?q?=E7=95=A5=E8=A7=84=E5=88=99=E9=85=8D=E7=BD=AE=E5=92=8C=20Agent?=
=?UTF-8?q?=20=E8=8E=B7=E5=8F=96=20IP?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
README.md | 14 +--
cmd/agent/monitor/myip.go | 103 ++--------------------
cmd/dashboard/controller/member_api.go | 23 ++---
cmd/dashboard/main.go | 24 ++---
cmd/dashboard/rpc/rpc.go | 15 +++-
model/alertrule.go | 77 +---------------
model/cron.go | 6 ++
model/monitor.go | 9 +-
model/rule.go | 77 ++++++++++++++++
pkg/utils/http.go | 106 +++++++++++++++++++++++
resource/static/brand.png | Bin 16017 -> 9805 bytes
resource/static/main.js | 91 +++++++++----------
resource/template/component/cron.html | 12 ++-
resource/template/component/monitor.html | 9 +-
resource/template/dashboard/cron.html | 4 +-
resource/template/dashboard/monitor.html | 22 ++---
resource/template/dashboard/server.html | 2 +
service/dao/alertsentinel.go | 31 +++++--
service/dao/dao.go | 32 ++++++-
service/dao/notification.go | 4 +
20 files changed, 370 insertions(+), 291 deletions(-)
create mode 100644 model/rule.go
create mode 100644 pkg/utils/http.go
diff --git a/README.md b/README.md
index 94e2521..ab195f8 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,12 @@
-
-
+
+
:trollface: 哪吒监控 一站式轻监控轻运维系统。支持系统状态、HTTP(SSL 证书变更、即将到期、到期)、TCP、Ping 监控报警,命令批量执行和计划任务。
+:trollface: 哪吒监控 一站式轻监控轻运维系统。支持系统状态、HTTP(SSL 证书变更、即将到期、到期)、TCP、Ping 监控报警,命令批量执行和计划任务。
-\>> 交流论坛:[打杂社区](https://daza.net/c/nezha) (Lemmy) - \>> QQ 交流群:872069346 **加群要求:已搭建好哪吒监控 & 有 2+ 服务器** \>> [我们的用户](https://www.google.com/search?q="powered+by+哪吒监控%7C哪吒面板"&filter=0) (Google) @@ -102,6 +101,9 @@ URL 里面也可放置占位符,请求时会进行简单的字符串替换。 - net_in_speed(入站网速)、net_out_speed(出站网速)、net_all_speed(双向网速)、transfer_in(入站流量)、transfer_out(出站流量)、transfer_all(双向流量):Min/Max 数值为字节(1kb=1024,1mb = 1024\*1024) - offline:不支持 Min/Max 参数 - Duration:持续秒数,监控比较简陋,取持续时间内的 70% 采样结果 +- Cover + - `0` 监控所有,通过 `Ignore` 忽略特定服务器 + - `1` 忽略所有,通过 `Ignore` 监控特定服务器 - Ignore: `{"1": true, "2":false}` 忽略此规则的服务器 ID 列表,比如忽略服务器 ID 5 的离线通知 `[{"Type":"offline","Duration":10, "Ignore":{"5": true}}]` diff --git a/cmd/agent/monitor/myip.go b/cmd/agent/monitor/myip.go index 4da05e5..cdf8962 100644 --- a/cmd/agent/monitor/myip.go +++ b/cmd/agent/monitor/myip.go @@ -1,18 +1,13 @@ package monitor import ( - "context" "encoding/json" - "errors" "fmt" "io/ioutil" - "net" "net/http" - "strings" - "sync" "time" - "github.com/miekg/dns" + "github.com/naiba/nezha/pkg/utils" ) type geoIP struct { @@ -24,14 +19,16 @@ var ( ipv4Servers = []string{ "https://api-ipv4.ip.sb/geoip", "https://ip4.seeip.org/geoip", + "https://ipapi.co/json", } ipv6Servers = []string{ "https://ip6.seeip.org/geoip", "https://api-ipv6.ip.sb/geoip", + "https://ipapi.co/json", } cachedIP, cachedCountry string - httpClientV4 = newHTTPClient(time.Second*20, time.Second*5, time.Second*10, false) - httpClientV6 = newHTTPClient(time.Second*20, time.Second*5, time.Second*10, true) + httpClientV4 = utils.NewSingleStackHTTPClient(time.Second*20, time.Second*5, time.Second*10, false) + httpClientV6 = utils.NewSingleStackHTTPClient(time.Second*20, time.Second*5, time.Second*10, true) ) func UpdateIP() { @@ -73,93 +70,3 @@ func fetchGeoIP(servers []string, isV6 bool) geoIP { } return ip } - -func newHTTPClient(httpTimeout, dialTimeout, keepAliveTimeout time.Duration, ipv6 bool) *http.Client { - dialer := &net.Dialer{ - Timeout: dialTimeout, - KeepAlive: keepAliveTimeout, - } - - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - ForceAttemptHTTP2: false, - DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) { - ip, err := resolveIP(addr, ipv6) - if err != nil { - return nil, err - } - return dialer.DialContext(ctx, network, ip) - }, - } - - return &http.Client{ - Transport: transport, - Timeout: httpTimeout, - } -} - -func resolveIP(addr string, ipv6 bool) (string, error) { - url := strings.Split(addr, ":") - - m := new(dns.Msg) - if ipv6 { - m.SetQuestion(dns.Fqdn(url[0]), dns.TypeAAAA) - } else { - m.SetQuestion(dns.Fqdn(url[0]), dns.TypeA) - } - m.RecursionDesired = true - - dnsServers := []string{"2606:4700:4700::1001", "2001:4860:4860::8844"} - if !ipv6 { - dnsServers = []string{"1.0.0.1", "8.8.4.4"} - } - - var wg sync.WaitGroup - var resolveLock sync.RWMutex - var ipv4Resolved, ipv6Resolved bool - - wg.Add(len(dnsServers)) - for i := 0; i < len(dnsServers); i++ { - go func(i int) { - defer wg.Done() - c := new(dns.Client) - c.Timeout = time.Second * 3 - r, _, err := c.Exchange(m, net.JoinHostPort(dnsServers[i], "53")) - if err != nil { - return - } - resolveLock.Lock() - defer resolveLock.Unlock() - if ipv6 && ipv6Resolved { - return - } - if !ipv6 && ipv4Resolved { - return - } - for _, ans := range r.Answer { - if ipv6 { - if aaaa, ok := ans.(*dns.AAAA); ok { - url[0] = "[" + aaaa.AAAA.String() + "]" - ipv6Resolved = true - } - } else { - if a, ok := ans.(*dns.A); ok { - url[0] = a.A.String() - ipv4Resolved = true - } - } - } - }(i) - } - wg.Wait() - - if ipv6 && !ipv6Resolved { - return "", errors.New("the AAAA record not resolved") - } - - if !ipv6 && !ipv4Resolved { - return "", errors.New("the A record not resolved") - } - - return strings.Join(url, ":"), nil -} diff --git a/cmd/dashboard/controller/member_api.go b/cmd/dashboard/controller/member_api.go index 09b5c50..c39ef07 100644 --- a/cmd/dashboard/controller/member_api.go +++ b/cmd/dashboard/controller/member_api.go @@ -15,7 +15,6 @@ import ( "github.com/naiba/nezha/model" "github.com/naiba/nezha/pkg/mygin" "github.com/naiba/nezha/pkg/utils" - pb "github.com/naiba/nezha/proto" "github.com/naiba/nezha/service/dao" ) @@ -196,6 +195,7 @@ type monitorForm struct { Name string Target string Type uint8 + Cover uint8 Notify string SkipServersRaw string } @@ -210,6 +210,7 @@ func (ma *memberAPI) addOrEditMonitor(c *gin.Context) { m.Type = mf.Type m.ID = mf.ID m.SkipServersRaw = mf.SkipServersRaw + m.Cover = mf.Cover m.Notify = mf.Notify == "on" } if err == nil { @@ -239,6 +240,7 @@ type cronForm struct { Scheduler string Command string ServersRaw string + Cover uint8 PushSuccessful string } @@ -253,6 +255,7 @@ func (ma *memberAPI) addOrEditCron(c *gin.Context) { cr.ServersRaw = cf.ServersRaw cr.PushSuccessful = cf.PushSuccessful == "on" cr.ID = cf.ID + cr.Cover = cf.Cover err = json.Unmarshal([]byte(cf.ServersRaw), &cr.Servers) } if err == nil { @@ -281,21 +284,7 @@ func (ma *memberAPI) addOrEditCron(c *gin.Context) { dao.Cron.Remove(crOld.CronID) } - cr.CronID, err = dao.Cron.AddFunc(cr.Scheduler, func() { - dao.ServerLock.RLock() - defer dao.ServerLock.RUnlock() - for j := 0; j < len(cr.Servers); j++ { - if dao.ServerList[cr.Servers[j]].TaskStream != nil { - dao.ServerList[cr.Servers[j]].TaskStream.Send(&pb.Task{ - Id: cr.ID, - Data: cr.Command, - Type: model.TaskTypeCommand, - }) - } else { - dao.SendNotification(fmt.Sprintf("计划任务:%s,服务器:%s 离线,无法执行。", cr.Name, dao.ServerList[cr.Servers[j]].Name), false) - } - } - }) + cr.CronID, err = dao.Cron.AddFunc(cr.Scheduler, dao.CronTrigger(cr)) if err != nil { panic(err) } @@ -318,7 +307,7 @@ func (ma *memberAPI) manualTrigger(c *gin.Context) { return } - dao.CronTrigger(&cr) + dao.ManualTrigger(&cr) c.JSON(http.StatusOK, model.Response{ Code: http.StatusOK, diff --git a/cmd/dashboard/main.go b/cmd/dashboard/main.go index 6680f56..ce3ae2a 100644 --- a/cmd/dashboard/main.go +++ b/cmd/dashboard/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "time" "github.com/patrickmn/go-cache" @@ -12,7 +11,6 @@ import ( "github.com/naiba/nezha/cmd/dashboard/controller" "github.com/naiba/nezha/cmd/dashboard/rpc" "github.com/naiba/nezha/model" - pb "github.com/naiba/nezha/proto" "github.com/naiba/nezha/service/dao" ) @@ -84,21 +82,13 @@ func loadCrons() { var err error for i := 0; i < len(crons); i++ { cr := crons[i] - cr.CronID, err = dao.Cron.AddFunc(cr.Scheduler, func() { - dao.ServerLock.RLock() - defer dao.ServerLock.RUnlock() - for j := 0; j < len(cr.Servers); j++ { - if dao.ServerList[cr.Servers[j]].TaskStream != nil { - dao.ServerList[cr.Servers[j]].TaskStream.Send(&pb.Task{ - Id: cr.ID, - Data: cr.Command, - Type: model.TaskTypeCommand, - }) - } else { - dao.SendNotification(fmt.Sprintf("计划任务:%s,服务器:%s 离线,无法执行。", cr.Name, dao.ServerList[cr.Servers[j]].Name), false) - } - } - }) + + crIgnoreMap := make(map[uint64]bool) + for j := 0; j < len(cr.Servers); j++ { + crIgnoreMap[cr.Servers[j]] = true + } + + cr.CronID, err = dao.Cron.AddFunc(cr.Scheduler, dao.CronTrigger(cr)) if err != nil { panic(err) } diff --git a/cmd/dashboard/rpc/rpc.go b/cmd/dashboard/rpc/rpc.go index 19a3fb4..2a3bfd0 100644 --- a/cmd/dashboard/rpc/rpc.go +++ b/cmd/dashboard/rpc/rpc.go @@ -7,6 +7,7 @@ import ( "google.golang.org/grpc" + "github.com/naiba/nezha/model" pb "github.com/naiba/nezha/proto" "github.com/naiba/nezha/service/dao" rpcService "github.com/naiba/nezha/service/rpc" @@ -39,13 +40,21 @@ func DispatchTask(duration time.Duration) { } hasAliveAgent = false } - // 1. 如果此任务不可使用此服务器请求,跳过这个服务器(有些 IPv6 only 开了 NAT64 的机器请求 IPv4 总会出问题) - // 2. 如果服务器不在线,跳过这个服务器 - if tasks[i].SkipServers[dao.SortedServerList[index].ID] || dao.SortedServerList[index].TaskStream == nil { + + // 1. 如果服务器不在线,跳过这个服务器 + if dao.SortedServerList[index].TaskStream == nil { i-- index++ continue } + // 2. 如果此任务不可使用此服务器请求,跳过这个服务器(有些 IPv6 only 开了 NAT64 的机器请求 IPv4 总会出问题) + if (tasks[i].Cover == model.MonitorCoverAll && tasks[i].SkipServers[dao.SortedServerList[index].ID]) || + (tasks[i].Cover == model.MonitorCoverIgnoreAll && !tasks[i].SkipServers[dao.SortedServerList[index].ID]) { + i-- + index++ + continue + } + hasAliveAgent = true dao.SortedServerList[index].TaskStream.Send(tasks[i].PB()) index++ diff --git a/model/alertrule.go b/model/alertrule.go index 82bd686..61e0919 100644 --- a/model/alertrule.go +++ b/model/alertrule.go @@ -1,79 +1,11 @@ package model import ( - "bytes" "encoding/json" - "fmt" - "time" "gorm.io/gorm" ) -const ( - RuleCheckPass = 1 - RuleCheckFail = 0 -) - -type Rule struct { - // 指标类型,cpu、memory、swap、disk、net_in_speed、net_out_speed - // net_all_speed、transfer_in、transfer_out、transfer_all、offline - Type string `json:"type,omitempty"` - Min uint64 `json:"min,omitempty"` // 最小阈值 (百分比、字节 kb ÷ 1024) - Max uint64 `json:"max,omitempty"` // 最大阈值 (百分比、字节 kb ÷ 1024) - Duration uint64 `json:"duration,omitempty"` // 持续时间 (秒) - Ignore map[uint64]bool `json:"ignore,omitempty"` //忽略此规则的ID列表 -} - -func percentage(used, total uint64) uint64 { - if total == 0 { - return 0 - } - return used * 100 / total -} - -// Snapshot 未通过规则返回 struct{}{}, 通过返回 nil -func (u *Rule) Snapshot(server *Server) interface{} { - if u.Ignore[server.ID] { - return nil - } - var src uint64 - switch u.Type { - case "cpu": - src = uint64(server.State.CPU) - case "memory": - src = percentage(server.State.MemUsed, server.Host.MemTotal) - case "swap": - src = percentage(server.State.SwapUsed, server.Host.SwapTotal) - case "disk": - src = percentage(server.State.DiskUsed, server.Host.DiskTotal) - case "net_in_speed": - src = server.State.NetInSpeed - case "net_out_speed": - src = server.State.NetOutSpeed - case "net_all_speed": - src = server.State.NetOutSpeed + server.State.NetOutSpeed - case "transfer_in": - src = server.State.NetInTransfer - case "transfer_out": - src = server.State.NetOutTransfer - case "transfer_all": - src = server.State.NetOutTransfer + server.State.NetInTransfer - case "offline": - if server.LastActive.IsZero() { - src = 0 - } else { - src = uint64(server.LastActive.Unix()) - } - } - - if u.Type == "offline" && uint64(time.Now().Unix())-src > 6 { - return struct{}{} - } else if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { - return struct{}{} - } - return nil -} - type AlertRule struct { Common Name string @@ -103,8 +35,7 @@ func (r *AlertRule) Snapshot(server *Server) []interface{} { return point } -func (r *AlertRule) Check(points [][]interface{}) (int, string) { - var dist bytes.Buffer +func (r *AlertRule) Check(points [][]interface{}) (int, bool) { var max int var count int for i := 0; i < len(r.Rules); i++ { @@ -125,11 +56,11 @@ func (r *AlertRule) Check(points [][]interface{}) (int, string) { } if fail/total > 0.7 { count++ - dist.WriteString(fmt.Sprintf("%+v\n", r.Rules[i])) + break } } if count == len(r.Rules) { - return max, dist.String() + return max, false } - return max, "" + return max, true } diff --git a/model/cron.go b/model/cron.go index d8ef23f..c03b7f0 100644 --- a/model/cron.go +++ b/model/cron.go @@ -8,6 +8,11 @@ import ( "gorm.io/gorm" ) +const ( + CronCoverIgnoreAll = iota + CronCoverAll +) + type Cron struct { Common Name string @@ -17,6 +22,7 @@ type Cron struct { PushSuccessful bool // 推送成功的通知 LastExecutedAt time.Time // 最后一次执行时间 LastResult bool // 最后一次执行结果 + Cover uint8 CronID cron.EntryID `gorn:"-"` ServersRaw string diff --git a/model/monitor.go b/model/monitor.go index 49ceb3d..4003332 100644 --- a/model/monitor.go +++ b/model/monitor.go @@ -15,6 +15,11 @@ const ( TaskTypeCommand ) +const ( + MonitorCoverAll = iota + MonitorCoverIgnoreAll +) + type Monitor struct { Common Name string @@ -22,8 +27,8 @@ type Monitor struct { Target string SkipServersRaw string Notify bool - - SkipServers map[uint64]bool `gorm:"-" json:"-"` + Cover uint8 + SkipServers map[uint64]bool `gorm:"-" json:"-"` } func (m *Monitor) PB() *pb.Task { diff --git a/model/rule.go b/model/rule.go new file mode 100644 index 0000000..9731fae --- /dev/null +++ b/model/rule.go @@ -0,0 +1,77 @@ +package model + +import "time" + +const ( + RuleCoverAll = iota + RuleCoverIgnoreAll +) + +type Rule struct { + // 指标类型,cpu、memory、swap、disk、net_in_speed、net_out_speed + // net_all_speed、transfer_in、transfer_out、transfer_all、offline + Type string `json:"type,omitempty"` + Min uint64 `json:"min,omitempty"` // 最小阈值 (百分比、字节 kb ÷ 1024) + Max uint64 `json:"max,omitempty"` // 最大阈值 (百分比、字节 kb ÷ 1024) + Duration uint64 `json:"duration,omitempty"` // 持续时间 (秒) + Cover uint64 `json:"cover,omitempty"` // 覆盖范围 RuleCoverAll/IgnoreAll + Ignore map[uint64]bool `json:"ignore,omitempty"` // 覆盖范围的排除 +} + +func percentage(used, total uint64) uint64 { + if total == 0 { + return 0 + } + return used * 100 / total +} + +// Snapshot 未通过规则返回 struct{}{}, 通过返回 nil +func (u *Rule) Snapshot(server *Server) interface{} { + // 监控全部但是排除了此服务器 + if u.Cover == RuleCoverAll && u.Ignore[server.ID] { + return nil + } + // 忽略全部但是指定监控了此服务器 + if u.Cover == RuleCoverIgnoreAll && !u.Ignore[server.ID] { + return nil + } + + var src uint64 + + switch u.Type { + case "cpu": + src = uint64(server.State.CPU) + case "memory": + src = percentage(server.State.MemUsed, server.Host.MemTotal) + case "swap": + src = percentage(server.State.SwapUsed, server.Host.SwapTotal) + case "disk": + src = percentage(server.State.DiskUsed, server.Host.DiskTotal) + case "net_in_speed": + src = server.State.NetInSpeed + case "net_out_speed": + src = server.State.NetOutSpeed + case "net_all_speed": + src = server.State.NetOutSpeed + server.State.NetOutSpeed + case "transfer_in": + src = server.State.NetInTransfer + case "transfer_out": + src = server.State.NetOutTransfer + case "transfer_all": + src = server.State.NetOutTransfer + server.State.NetInTransfer + case "offline": + if server.LastActive.IsZero() { + src = 0 + } else { + src = uint64(server.LastActive.Unix()) + } + } + + if u.Type == "offline" && uint64(time.Now().Unix())-src > 6 { + return struct{}{} + } else if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { + return struct{}{} + } + + return nil +} diff --git a/pkg/utils/http.go b/pkg/utils/http.go new file mode 100644 index 0000000..3ae3a4c --- /dev/null +++ b/pkg/utils/http.go @@ -0,0 +1,106 @@ +package utils + +import ( + "context" + "errors" + "net" + "net/http" + "strings" + "sync" + "time" + + "github.com/miekg/dns" +) + +func NewSingleStackHTTPClient(httpTimeout, dialTimeout, keepAliveTimeout time.Duration, ipv6 bool) *http.Client { + dialer := &net.Dialer{ + Timeout: dialTimeout, + KeepAlive: keepAliveTimeout, + } + + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: false, + DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) { + ip, err := resolveIP(addr, ipv6) + if err != nil { + return nil, err + } + return dialer.DialContext(ctx, network, ip) + }, + } + + return &http.Client{ + Transport: transport, + Timeout: httpTimeout, + } +} + +func resolveIP(addr string, ipv6 bool) (string, error) { + url := strings.Split(addr, ":") + + m := new(dns.Msg) + if ipv6 { + m.SetQuestion(dns.Fqdn(url[0]), dns.TypeAAAA) + } else { + m.SetQuestion(dns.Fqdn(url[0]), dns.TypeA) + } + m.RecursionDesired = true + + dnsServers := []string{"2606:4700:4700::1001", "2001:4860:4860::8844", "2400:3200::1", "2400:3200:baba::1"} + if !ipv6 { + dnsServers = []string{"1.0.0.1", "8.8.4.4", "223.5.5.5", "223.6.6.6"} + } + + var wg sync.WaitGroup + var resolveLock sync.RWMutex + var ipv4Resolved, ipv6Resolved bool + + wg.Add(len(dnsServers) + 1) + go func() { + + }() + for i := 0; i < len(dnsServers); i++ { + go func(i int) { + defer wg.Done() + c := new(dns.Client) + c.Timeout = time.Second * 3 + r, _, err := c.Exchange(m, net.JoinHostPort(dnsServers[i], "53")) + if err != nil { + return + } + resolveLock.Lock() + defer resolveLock.Unlock() + if ipv6 && ipv6Resolved { + return + } + if !ipv6 && ipv4Resolved { + return + } + for _, ans := range r.Answer { + if ipv6 { + if aaaa, ok := ans.(*dns.AAAA); ok { + url[0] = "[" + aaaa.AAAA.String() + "]" + ipv6Resolved = true + } + } else { + if a, ok := ans.(*dns.A); ok { + url[0] = a.A.String() + ipv4Resolved = true + } + } + } + }(i) + } + wg.Wait() + + if ipv6 && !ipv6Resolved { + return "", errors.New("the AAAA record not resolved") + } + + if !ipv6 && !ipv4Resolved { + return "", errors.New("the A record not resolved") + } + + return strings.Join(url, ":"), nil +} diff --git a/resource/static/brand.png b/resource/static/brand.png index 5eaea208440db959410e25340836e6d57342a50c..7e760ea07953488b83c50dc2670b045dce02ba88 100644 GIT binary patch literal 9805 zcmb7qcQjnz7w?EfB%%!l6D@j~A;O3VB6=4kF?tLUMjgF|s2QUZBubQM5k&7q%qS5p zqj!VoWpvS=e1C7P_xD@t+_lTupS|~8_pWo#KKr7dX{%7)2Hyq%0F-K~ih2M5DGmT2 zL6DPPX;zSt_pS(uo1Th1$#^#;@JhJlqH5#@06;YUT_kxj$wgPn#AliYN>`NHrS|`8 zK*CF)Zwnxw@KEajsB<*bHUxsHhNgBxUIjuuQy>Y=5T7)NVO& z%q@U$J<{JjSegcqYF$j+b}kA80~+LOuHaVQcL1^eFxkH`PA~xE!3fhd65wt9l{k3s zkQNZ6(066>`N9LRa^1-OjI>u2aHnE~>58!IjvU`~tq0ihIY$Rt7104U7fVF{#E`s< ztMBL}cO@B6{{v>fcjJ1{3)fMmW+cG4+ssOnm2^OT3w+se%Mnl&-`LRwAjx{a0lPrA zi2|V3WkhP*J8~4`_GfoRg#pH07FHWer2l09(5fr^mAr@|8IZpKUuvGyk;FY7U<&FZ zjV4)<{>LP?c0_bt0 F(Q@q*dC*k1g;i8EiS5%|Vr+YRV#p0Yp`Vws zkEG9NN9S(;3zyT;N#Sz@PE7sp#bHyh0z5#J5Mw{h-WlQ8IG1`9+5xcsQI q*1J6qU2iYkf$<6N8 zc 1`U{GO%F4pLfcRrqnlDc;GXIVM##rCn___1La}R*E648_=|C0uI*EQ1*0C7E| zlrw0#PVRp1#|=5tYx~8t^Vc{Y_{}Vsb`_FwNVNL*KeoI|+@U4&xcZpsD&L&S?OK!Q zY*E0i<3H8^mm!E5M;loI66Pu&UBv&j0gP$2c(br_WrKsnS&s2c0Z72_Z_6cy(XucA z`grYsOwmrw?_2+7=E_8sKY@d|j}w{d*#X>b1z45NqHVIEpv4B$@5NZvxoZJ7mtL8Z z`cDyn)Q!tNx0FY!cL0g!f|Z<4qVI_UG!G3|b?fc!ph;d bJ(tuN#p()3j@t4~9$SZ!EN zgFXqdYc7iF5RSpy->_*T;iDs_I(kFtJ*#2xK)yHO#U-GgKi(_mCMMD!`9DcFstuqC z>MSM;tjWi|`Y>cOG6442tf~2(DR$6>m(f-?pxNEU1lpq8TEAi5=wA@vZm$XA^>j!O zL7DzbMz=Y7(m>l0W+E}A8)8{- mlOW(17} zLEzVeq34wz+ov6;` ooyD|O&ukfo40n^q>dc>a(?EGC<2DET2 z2A^CXbuoDpL~QvkTDorMmo1XY>UR_KeBq0TX9FxB$K~3j!N5=053a|e#J7sE_)5q7 zAIPazAF&g`+nn%0kqF9j+w8)o^+#~c*zReY;t!zN%xP0vw;^nM^zpF=Q>7z!Og5^M zCsW-abGG2UZLzvsEx#0~898zqjI9HaSyTlcd^8z>u|o16Uw6_a$;b74npdVWGZUn) zL;oGOOl^(Hz|OaY{{ht`~%Bju-S#yQ&jFQZpkI#=>-znF|FlNo;Vcm0-nrdLY^ z!xIk3;dTb^pJ+b~47nT#E5T;Nwi>X;m{G^B3d0igpzZv@>t1~kCy9Bn4};!P3vY^_ zUDL)`Ig2uDA`SsnI}r1urC~d6AOFw(zUR9A%}76VJ95eg?ExD7!_YGl`()TaG+Rg{ zjAo%huOdN-?{ Lr<_qU1U?7VQN#)nC6IU6JC$28Z)^X3UBedqK{V)ml4K zkhT158Mp2k#Dh8!A(Q-v4`Ysa;Gnx&LgI5{r?Yc{DE{bE4pIGDp+qTBYm39Lyw+;D zIQiv$f1_1OzrW&}<-C5FqU;7Bu=K0=yL}YoaJR=G-EZ}IqXvueg1M^0$a9;v-oJ%3 zeh#6yG%Y)A+^AvNmSl}&;`n}&Wo$L>v&OD95t%Aw`C7~kGO`#mnT6&9&npmUz()#* z7Tog`9p4hL-p}X$4lYJ^X7TZf!zuVdE-KBpH;14VP+kmQ;JUJ;lWMA^Xf@d&+p>8z zJNn#yiVjUVLHO|3$q++Pbl(PQKYp|NO9}5a>OvE16HVy71S^y3!;j2B8~6s#%Wt@H z!%YpuEci$XL5e}o3(pc>!4g4y{G@vA7pO=15*v@q)9>Dv&T@)(6!lPW?W})uv#C$0 z9Wna7ISQ|?9tb=gu6qdFgR%j)C92_(?n2|(j@+p|+Q;m#JY}1gwvPo7j^(T8>E(w{ z?P|x~WM#MCi|r}E%EGo{u^YZGp8Z4l>e_k#vU;8 9A2bXxv!%Ur+XE7eAF=*}oHykJ{6trI!_y)T zXXqb?76oW*Xe`L8{i-?+pZU=~fUv7jHOqW$N&L#j^3hDI)8I7aC!&u|InU#cx{)cm z(X-b}e}iUfSEQUh+K)#RPoO<_Qo!U*6v)m(eTPy)mfUv_5!gmOBZr66cqwsSh`Fa5 zucy#}@tlrzacJCFM8|moycR6)`+V;Ths#H+et1xuTKE#%2sHX-L8n(B^Io(1*z=&N zQ2g+*;f8n*y)wcp+K(4yfWgtg%4frTqCTQjUUx%k?CIX;k6CZk4#CquOSE{|F@2w+ z3gb?~mV%G82!$Yw2rUVnKBhaM^4$?n#TXL)RRUMA$v)xA)!)qQ?95xUI91Zk|3hsf zAGOULV^==b_E{<>AC#DwiaiAyISf}mFA-Z$omI_-;?d@-lPs=<3hEm-KLU% S>X<7^xS#i{sFREjUUrfz2KxtmWI%WaqG)52s?Ih937x+@H7i7$ z5GuCzlG@GsXbY+>W6#S#8VLR9Q$ds>CaeZ6OZa?c*!}D9rv9Tvr81|0ab4a~naJjx z%?-Gv*_1ah%`UwR>viPxDdlo)9s-gda(`W-0}|TPq`E23lo$KXmktbG@tbqxK`2 z#=41`!AZTzZ3-HF1YNXP^6~uO<~3K=EU3h|uEJ|nvmlU<^! >86VQ0 @==+|#aCC(E~VT6<2%rNou4MX+`5(2jLESn z?!brkQzYo+GVRg9p_}!;!CwtV;il_}j14~&E4|IiKWxU- yP>PXiIXO1$dl(*HHlbm$eyx|A%F1D=4o!d@@062f^4wm!jypL~Gy|^&=KJo=$wA z-tu_d$LD>Mos1b2-+LNL6JH>jJEr5jx00Ikim27|^4722a71ZccZ`rq#vu~Dhi5jJ zOdv{ynBGZrcs%hO$O|g&4Mfv~j(px{iiy?N_=>ON0yf?#f*m|9syf3}RN<=slnsD* z6L1MPHVeL7fvJ@Es_%nERhvaKMn%h0?`WutsWtp+`jMr%%99V wbHe=}$z}+iT2^}$YeAExh7KaghudDU_8GK)M?6^7_HkGd|8w-kEiUzU zofxn` }9^%*0=cQ+!?rDjp%5IaLrNCW} K-^fdjal zk1{|f8u@EA7uLdmdOlO>$cAKAQwQ9=Wl8)ZS~LFFsFuaM_n<+kdP25%Ew #t_P`LWLkOG`V3Y%#`UNrnqnq?#LQq?4WWYDK?(5 z Oo}R(ebJgnC!(zC*SbrMJdKEDv!m{Lq_q*JTKbCxaiGdO zr}|G8O^QhPXP1>2 qge1t#$uSflZ_>DmNxP~%lH1k*iV9F@sb)tQ{zbrj_| z;$qwzIz-A?&ZC`;l1E!DIuS2-dO;dyZi8j4HFc M 2gh*s|c-F43Z(}JE9#PO7NsPV* H5e|;jN |R5iG@6i@7XqL(?v{1h zg+F%I=mHc1&1Lj;^-ep&YXYQ1?z3TtY*jsJQX@37q4HNHqao$u+E5fXhNp7`b`#Wy zw?UO?n3=lv*&(Uz#p3e6QPXt;RTdV3Diz-HO%nI3;cLklQu!YQB3X{l{ETkbFinM? zN%msn;G0lX>Q7{_lb++8CGmuEEw+7z{}%=)QGp$hY{trZRpbfG7HMRPSHer*%3Eqy z9@l#f1z@het|Bk&KC;$~X(#W13sT0`OLxVW9^-QBYBWZFL6Yez!hU>5u;x})?)CD+ z;&D-9TfjG9dL=?ilNGdj0Xw0!&6 +a3GjI65Q$xM#`Vz^Ctz z`_u`lAWS9eWLHEyPTl)Czosds;Ppd8g%L7lW;g^ib+NY=OV^KJG?GVdTpcqdX5eIX zWb>H$OI9S>MzDnJGhbtj2JiCb?e7GjVf>W!t%CB`=$p>WCXgE`wLYz6Ai58s@|FZ+ z|C!@5kC&)!b7LcUyA2aUTW$ufpUOgQBKv+^ 1R@AGDetI&tE~H@Ov>S2iY5^HI$F4Cym2xO_%y|P=i>}w z8c8m|Tv!v=*G(9iDp9^FQ}WA?D~N*EzSVKoLoQ%c9K`S1h?F5{4eoEq?{tWh*@rRm z^_V1QBh-X90}0%e$W;$%fl=Kf26TWhdzuH+kr@yW SL`8_E0p@kIA#D(jfseV1pufNYoD45COOUMtVOWutj;$D6 z7(BJG_}<4dP5?#-f|ot}Nzhri51b^cHO ;FX{3bF;hndrS^XNM}>b)eOk<>p&tpqh|g z( ZRWlEVLkz%%O>&$Ia>N_hjs*cevmL;h%puIP8q`{%7-`NDwRdvZg0 zeLsv$#V;#W=`kYXd^CKOW8$h;d#8udr0NcjA#ZNI$boTX(`HcRGc2Hr_Noz?GX&qL zDCrZ~9NZ74Y^`8^Q2&@o0(a$ctAjP_x1@EwLj+EnP74w$yO$&2G{ahN`HJu-
~3({fdV2w}zeq+CX2*YExk4pRIA;kaSC7^xEzo?zVQ zJ{`|pBeFwq1a#Cw1G6m;`l49a&X`+c74qe_wqa!tv5sg^{tQInGrPoERB28i25Gf^ zKjamXHfma9nG(N{rTny 1i8OnmCcUefV zNCB48x4Q0d?8V=s*2=h7m=7IoJqHf ?Xfn_;o)GMm0sVZD5Hri(G6 za#E?oq)fB|gXu`>`(_3mgjjTYN>7c#W|b0vG+UrF!gts`mZwLhvF2k~XCFe~(bO9F zBCfH*0dvl1%98I^QHxM_$Uz;u8hI5yA!YnJ^G7{E 8lf}mIAOTsRgJgS0lCodg cF;TsGUp!5NL+1gi+7ETIc1W`dK8WwG4)kx1uM`hzb8Wn}ZZ$x>Ch zQ|YdKt_Dfv7$^CcT8{LrInoEEPw`PGLG+|}jK1)CH{qMcJ>}}4*H2rlrt@0(Kfg^k zrA8P#%rMb~+~}F272%f&xsi(xm-GlQVCQTwr#^@MVk$_iGg?qxs)%t6%Yc?fcerc( z0g i zY<7&56mq )SDbnYvu33y!x1;8{e2PS2cpU{jjL`?+PQjEVr(KGR z8_-eP#EQw5K4tmRTPP@7l%_HF?w`*GG@^EJQ>7=Fp@UJ=?ml$C`OPY6ZIEb>f56S3 zgp)HF_rVuz4 &yN`?Jl&$HJ{3?OtRj@+aoc0XC!ZnL zWW?qfCo{b1<>ylOsr%`f{iRNi|9q2*PM#BRPcO!P=^*}mHA}BDrZbZ)4cX%Q@vc>s z*!ab4&6GPC7Go$JkuP~{BXihAdl%32aG-ouPp7Zi4;`+ws<+HlUWmfFJ`%^h9Tj`a zBA{=~iz%1;u}na<07qdl@PE)Vwi=En%DrhMopan)S0!pEBt=0#9TfrI|8SnpV{Xq< zX*!|-TwZw=V!*NtxvD1%mnw87Z-3e)0rWl{0j7XepeX@l9!W{m0REOmC{cc)`JIL5 z>*iEDwuK}Zz?Nz+SyFh!?@zB)djIvH@MDtC{-_VHB3&Fr0pOO4aI8Ixyh+=`*6Ues zKgD_DA~4ug2-&Ui2&~4|j2R@RBK;PCa`qky$IghRoxH{&Bdm{m=SUJqa-@B32MGmm z$Y>Mo7Oxa+rvZyX_j|9)AvbE9zj|w#y#aW#6`ZX;(kmoUjm2~8ueg~30GoA?VE(p4 zK-TG$jC6r;FolyCuIm^OFt?SB=V}kUl58WNQuFg - UznW&8V$!NncaQ^)?{=ID$k1%u%KCR2C?Bcd??nfwPSuNE2Y86n zm1K-JbVOjc;f8k_q9jamxOx+*(T#gu*MhR{Efvv$8-=aD98dNtG6Dp_dmxM5vZ<|Z zHlDE%uzPCq;=)v-==Pavlo9WMrLiH6v~}~3?f??S8}cSu?e`(x{oZT`pVsxT4CIb) zw|?0FtjLgj<2FtI@`l9Zwdt@)_Fi#=O&&?Jgo7-SAl~DUpAmODqrIF| E d=kF7cE;T^mJcvCrdefEsb5_- z=6;KS&ng~!rO{+VyV&3+jaA~#Wg`Rdj^#@?zm8)g&X?coD-0wFVy5!JP@}D$%Bo=9 z7Ii}U2+96uA6yWB`mAgl7FiQDv(oBHIlOg7D9(2zRGO~+Q+UJ4_8+Qk2JTLN$h4xp zz9RT$WVc!Q=y=B7+SiA!e47Ly;%yvv*KcHXrr!o|t*hz!G&T)OVV+and5>jlhotOf zEm43d{i5}+KxjL2*+;UhTm3*0Z}P1lX-BCez|T4%+5`)N&xbw+mmvS66+b6CN=}ok z#Ov{Oi#l1jPG`%8-#qoso3{X3l6-%*Kg}p8UCJuJ+TAvMZb|h#DeED(zW7Uja`DKZ zHT&AU-GI>#q-o>^j}2=TL#doM*qu?NGXb`Hu{>T>>3c~89_xxlV6!GeN4?u-Q(hOd zt+&Re3`D!!_>(GE>l2f$*1f(Q@%HEC44XxJZJWk}Y16v^2@=v_LFz!ZCz)Wsu_GP! z*uY@r3TqD06 Kvzq`lB$7faZR>W43vcZqD;5erJx7WeTq!lu9PsS{-y5=hs zkBddQ$<8fRuA1CgIvby_n%nyr)}iAYz>v&xH?URYt$phL)5>KkZn6Mwb!`t}4lnS2 zBI|Fa;#4_{Tg@|)j5AkWL@sl_Ue~1bJOniJj)nU&6N6-G l+it(grn{ B0n{avrbw zM}>V3i`nPy1IVObrSu&{rh)+BQ-ObyZU&!N0vzg2ySN7qj+l$7Nxtz!#2m0MYLL!h zr`>*s0R!0w-;;gM2rN?W+7@|dOEQP4O*!Vw<)aLGofCO>u}n$=1zg)XWahCwzX*Q8 z6y$Q|N0mNuu=DPD1mODerr5Kwzw>)nO=kcn>gaH_UMsu}z&*EmbpnukcGS;hlLS!h z#P2N^2kiVkSUS0-bsxa&a {|{W=_o4s* literal 16017 zcmYj&by$_p)Au1Hqz;I59HbjWy1P51K}w`sLJ*MhP$GgLCEY3ANQ)BEEhSyjAn@$L z@9%owKfLhl&d$z!W_EV&-PyMqYVtVu$?iiS5FAAX87&9|r5gf4vcNzCEsNE@55ONx z7X<@%2!u@u@ek>xuZJoG0)r^ZNa}c}?#`>~%}$-39cmkl_ugVcULq6l25Yn&FWU`= zMZs)PX=nqZAhx3z#~b%zKUXlg1YwGJqxwwH7&Ux7mtB|r>2nb r-hxJ)!QG z;eT^}ag>j2!7bD}udB3M=3PHDluy0q!;J(4sg`AuE0^o7qs<*l|C@H1kxhSVbo=G_ zZ@rDs6aJgq`S-W}x34ajE_R*o0qS=wQryktl#6@H74z{$PS;-^^p6DSD2YA^*vq}| zKe>E{42U8yGD#)?OTXw}P1FVlHM;1(n}nV`NCeXbdgH@KlxQK7;`@Sb<==C%N?z~j z&tii?T|pXAxjFY5Z$@amBWbN2SRSqkPq3nZo@b8;SGuJz^6t+?C!?;~8 @>I!EzE2^fD7~la^S-sj1^!|j}TFcB0mN#!!%N^kF25(Lf|exw}`f}YF{Dr z`U@kJckp;6{2nUuEZW8Em~I;jv(MoK6?Ex8B9vE{$=)vF+8wF?;PO3?12ncnGX)-H zuguaGebtw)dz#UAX<5_@+9j|;mfIC0k7WW1?q^3gRO@9_-g1NX3aU&2^XwHPN(Wqb zaa `vHw0aU%@ZT>fWVngrZj3$89qZgU_xkz_!TehRp?nma z+c%3*m%4I+1Q0U=Bx1~?sOA*}@^Z;QL;q)gA-8I+Y QgVG%K$87@KSAp@CdOG^-~hBnkR~j5X>j*IY^o-biV(79N#_c1>qKz}zQ6lv zA)70p34TINc&WrQ0=$1A98>}uu!qCB(Xxtr&fK&ShR2`+j5N7>0;eVs0(Q_6J`XgH zLK0OH=9R7o4N$t7d>#o_UbJdj>3W7{48#!pgykqOce`yqLS&Z$!sj7?=cy*MCqGz3 zE1`f!f(fHLnTEvDJQsC^n-k##QW8vq06COdEhT?y%Ej~pDbQ$cTf#g@VjqPcsGK_% z;@Jb5NbY9}xT9L4n&*9(VI%=2F~_J@e;%V}hbmFd6JQP@1~g$3OlTb>G*j=9jIWvz zFm61^B;my9!3f0d5zon`Mzq|^Bx%Es!jCu_Iou5)Ml2P7^I@U|^K&9(*Naz|L3bz# z$>x$cYjDfKaOHdJh?@}zY!GCUD1(6*!3b1^WSM%DfHcA8U>ODuR^*Y8SN4zuC9&l8 zfGCmfBME4JVP%q(LcvtYT6P%-I*6HVh8%0P7A-VaYwOe9+^taf2w4Y7koORoB_KGL zx_T1jIj2-QYq>o!b=7lQHt%?TeT8-VI_JZT{onnci_+cgQ@sdz1~IxTmZ4G930ip3 z?N ypO7z+-0N0h=|M%nR3Zx+^U&iJ1=2M{_r`u4ZHBvk9YO>$DkwY%sXtLqh~i^2 zZrb@J2*Pw7cZ}4*1Q07~E6Rn1BVX@cJ+O5^^xtY=qsn}(ETf|S)9BqPV~5}^!-R%R zw;eM?@@)<0O&w^d7!$_Df~F(XeN4CIoICd6V_&lA7ga1oq*?f^g)Jj7Q#3@ yaPDsFq{cNjD&e5uz}^V^8%sITGcxwcjyA zxYkU55MdvYny=JJWr(Oltt>f0IH0mr4yAW&O;2e8uvokT6)R{JlDK;EjtSGx%-i4S zSqd;qpo%6l8o|k{OUdx%wio$Mp?m;{Koy`th-gMYeY}74i+t1Ae~lwzI93)OuZoK^ z!^?YHt88 zfgG$>iU~o4k?Ac#pdY9PjVRqvoL|A9<_9Ef_;B>cLapQ5!qkAJya5%&11{kpi@xlS z<_JJs4b>2VV0H;rI56i9V)h)om2e1^006vWmU0~eOE}sbn-il;52FjPBBdofvz3LF z;YTfn{Etjf%2k~Ybs4NVK=Y3bMtGKKji7q=KdM0DkmVRsGz3*C5DppvCO#m~td~v< zs+Z&HF+{asMYRAEG)pn *Y zgSk!r$F71Pb1FLqA#bW6a^*v0L{OQyTv;E=prFAM!tGMcNLLp(KE9aa68Wz?FguH> z_tH&=+0#V#%Q|5!W9HNwDij0*{Qr-_Gk?oxMZF=;B~^!rhESLchLD651~sCAqDX|V zKDv-$_SDh+A{@i(rbW0`re!zTsojd#veifj<=@<9Pm>)ivODGaPJ0}#v18sYVN6TH z7==(Hs$e6sn2>|vUAP>{LL6g>xcf_O$9xq<%@3EKEZ8KY_7%`&HMh>%iSb^KEz&IN z%s?ZE!EP3F$}pXQBofAS4;qnxy{Lt{a(ssX{=)`^7FM`c_%e3~GR^xVk4NH>lOjfc zOlo=~iJM!5ei6zFew^G@lX^TJ&}=~xQ4@Qa&-5)hWM!iYGr r`)1t zOqe?;cq??n9ea61o9Cj!#7Ah3u;c>?;GQr!4|tY{wvxZ;pZYev@tu&Tgwk=$U*dak zQfhAsi>7%w=Zs`EFYBk$^~N0*m$;MA8Vo4k(bIkTF1
* * * * * 分 时 天 月 星期,详情见 计划表达式格式&&/& 连接,如果遇到 xxx 命令找不到,可能是 PATH 环境变量的问题,Linux 主机在命令开头加入
+ 命令:就像写 shell/bat 脚本一样,但是不推荐换行,多个命令使用 &&/& 连接,如果遇到 xxx 命令找不到,可能是
+ PATH 环境变量的问题,Linux 主机在命令开头加入
source ~/.bashrc或者使用绝对路径执行。