From 9ad87292182079c1ccd91b500d3174df42ac9daa Mon Sep 17 00:00:00 2001 From: Juan Torres <11037563+JuanTorr@users.noreply.github.com> Date: Sat, 13 Jul 2019 02:40:17 -0500 Subject: [PATCH 1/2] `stats`, `stats items` and `stats slabs` added --- memcache/memcache.go | 10 +- memcache/memcache_test.go | 52 +++++ memcache/stats.go | 474 ++++++++++++++++++++++++++++++++++++++ memcache/stats_items.go | 380 ++++++++++++++++++++++++++++++ memcache/stats_slabs.go | 244 ++++++++++++++++++++ 5 files changed, 1159 insertions(+), 1 deletion(-) create mode 100644 memcache/stats.go create mode 100644 memcache/stats_items.go create mode 100644 memcache/stats_slabs.go diff --git a/memcache/memcache.go b/memcache/memcache.go index 25e88ca2..3c108ee8 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -48,7 +48,7 @@ var ( // CompareAndSwap) failed because the condition was not satisfied. ErrNotStored = errors.New("memcache: item not stored") - // ErrServer means that a server error occurred. + // ErrServerError means that a server error occurred. ErrServerError = errors.New("memcache: server error") // ErrNoStats means that no statistics were available. @@ -61,6 +61,10 @@ var ( // ErrNoServers is returned when no servers are configured or available. ErrNoServers = errors.New("memcache: no servers configured or available") + + // ErrDumpDisable is returned when the "stats cachedump" and + // "lru_crawler metadump" commands are disabled. + ErrDumpDisable = errors.New("memcache: cachedump/metadump disabled") ) const ( @@ -307,6 +311,10 @@ func (c *Client) onItem(item *Item, fn func(*Client, *bufio.ReadWriter, *Item) e return nil } +//FlushAll Invalidates all existing cache items. +// +//This command does not pause the server, as it returns immediately. +//It does not free up or flush memory at all, it just causes all items to expire. func (c *Client) FlushAll() error { return c.selector.Each(c.flushAllFromAddr) } diff --git a/memcache/memcache_test.go b/memcache/memcache_test.go index 4b52a911..1ba75a64 100644 --- a/memcache/memcache_test.go +++ b/memcache/memcache_test.go @@ -201,6 +201,41 @@ func testWithClient(t *testing.T, c *Client) { } testTouchWithClient(t, c) + // Test stats + if stats, err := c.StatsServers(); err != nil { + t.Errorf("Stats error: %v", err) + } else { + for addr, s := range stats { + if s.Errs != nil { + t.Errorf("Stats server %s errors: %v", addr.String(), s.Errs) + } else if s.ServerErr != nil { + t.Errorf("Stats server error: %s %v", addr.String(), s.ServerErr) + + } + } + } + // Test stats item + if stats, err := c.StatsItemsServers(0); err != nil { + t.Errorf("Stats error: %v", err) + } else { + for addr, s := range stats { + if s.ServerErr != nil { + t.Errorf("Stats server %s error: %v", addr.String(), s.ServerErr) + } + } + } + + // Test stats item + if stats, err := c.StatsSlabsServers(); err != nil { + t.Errorf("Stats error: %v", err) + } else { + for addr, s := range stats { + if s.ServerErr != nil { + t.Errorf("Stats server %s error: %v", addr.String(), s.ServerErr) + } + } + } + // Test Delete All err = c.DeleteAll() checkErr(err, "DeleteAll: %v", err) @@ -209,6 +244,23 @@ func testWithClient(t *testing.T, c *Client) { t.Errorf("post-DeleteAll want ErrCacheMiss, got %v", err) } + //Sleep to allow the memcache server to refresh + time.Sleep(time.Second * 5) + if stats, err := c.StatsItemsServers(0); err != nil { + t.Errorf("Stats error: %v", err) + } else { + for addr, stats := range stats { + if len(stats.StatsItemsSlabs) != 0 { + keys := []string{} + for _, itemsStats := range stats.StatsItemsSlabs { + for _, k := range itemsStats.Keys { + keys = append(keys, k.Key) + } + } + t.Errorf("post-DeleteAll stats items in server %s has %d item(s), key(s): %v", addr.String(), len(stats.StatsItemsSlabs), keys) + } + } + } } func testTouchWithClient(t *testing.T, c *Client) { diff --git a/memcache/stats.go b/memcache/stats.go new file mode 100644 index 00000000..4537be4f --- /dev/null +++ b/memcache/stats.go @@ -0,0 +1,474 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http:// www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memcache + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" +) + +type errorsSlice []error + +func (errs errorsSlice) Error() string { + switch len(errs) { + case 0: + return "" + case 1: + return errs[0].Error() + } + n := (len(errs) - 1) + for i := 0; i < len(errs); i++ { + n += len(errs[i].Error()) + } + + var b strings.Builder + b.Grow(n) + b.WriteString(errs[0].Error()) + for _, err := range errs[1:] { + b.WriteByte(':') + b.WriteString(err.Error()) + } + return b.String() +} + +func (errs *errorsSlice) AppendError(err error) { + if err == nil { + return + } + if errs == nil { + errs = &errorsSlice{err} + return + } + set := *errs + set = append(set, err) + *errs = set +} + +// ServerStats contains the general statistics from one server. +type ServerStats struct { + //ServerErr Error if can't get the stats information + ServerErr error + // Errs contains the errors that occurred while parsing the response + Errs errorsSlice + + // Version Version string of this server. + Version string + // AcceptingConns Whether or not server is accepting conns. + AcceptingConns bool + // HashIsExpanding Indicates if the hash table is being grown to a new size. + HashIsExpanding bool + // SlabReassignRunning If a slab page is being moved. + SlabReassignRunning bool + // PID Process id of this server process. + PID uint32 + // Uptime Number of secs since the server started. + Uptime uint32 + // Time current UNIX time according to the server. + Time uint32 + // RusageUser Accumulated user time for this process (seconds:microseconds). + RusageUser float64 + // RusageSystem Accumulated system time for this process (seconds:microseconds). + RusageSystem float64 + // MaxConnections Max number of simultaneous connections. + MaxConnections uint32 + // CurrConnections Number of open connections. + CurrConnections uint32 + // TotalConnections Total number of connections opened since the server started running. + TotalConnections uint32 + // ConnectionStructures Number of connection structures allocated by the server. + ConnectionStructures uint32 + // ReservedFds Number of misc fds used internally. + ReservedFds uint32 + // Threads Number of worker threads requested. (see doc/threads.txt). + Threads uint32 + // HashPowerLevel Current size multiplier for hash table. + HashPowerLevel uint32 + // SlabGlobalPagePool Slab pages returned to global pool for reassignment to other slab classes. + SlabGlobalPagePool uint32 + // PointerSize Default size of pointers on the host OS (generally 32 or 64). + PointerSize uint64 + // CurrItems Current number of items stored. + CurrItems uint64 + // TotalItems Total number of items stored since the server started. + TotalItems uint64 + // Bytes Current number of bytes used to store items. + Bytes uint64 + // RejectedConnections Conns rejected in maxconns_fast mode. + RejectedConnections uint64 + // CmdGet Cumulative number of retrieval reqs. + CmdGet uint64 + // CmdSet Cumulative number of storage reqs. + CmdSet uint64 + // CmdFlush Cumulative number of flush reqs. + CmdFlush uint64 + // CmdTouch Cumulative number of touch reqs. + CmdTouch uint64 + // GetHits Number of keys that have been requested and found present. + GetHits uint64 + // GetMisses Number of items that have been requested and not found. + GetMisses uint64 + // GetExpired Number of items that have been requested but had already expired. + GetExpired uint64 + // GetFlushed Number of items that have been requested but have been flushed via flush_all. + GetFlushed uint64 + // DeleteMisses Number of deletions reqs for missing keys. + DeleteMisses uint64 + // DeleteHits Number of deletion reqs resulting in an item being removed. + DeleteHits uint64 + // IncrMisses Number of incr reqs against missing keys. + IncrMisses uint64 + // IncrHits Number of successful incr reqs. + IncrHits uint64 + // DecrMisses Number of decr reqs against missing keys. + DecrMisses uint64 + // DecrHits Number of successful decr reqs. + DecrHits uint64 + // CasMisses Number of CAS reqs against missing keys. + CasMisses uint64 + // CasHits Number of successful CAS reqs. + CasHits uint64 + // CasBadval Number of CAS reqs for which a key was found, but the CAS value did not match. + CasBadval uint64 + // TouchHits Number of keys that have been touched with a new expiration time. + TouchHits uint64 + // TouchMisses Number of items that have been touched and not found. + TouchMisses uint64 + // AuthCmds Number of authentication commands handled, success or failure. + AuthCmds uint64 + // AuthErrors Number of failed authentications. + AuthErrors uint64 + // IdleKicks Number of connections closed due to reaching their idle timeout. + IdleKicks uint64 + // Evictions Number of valid items removed from cache to free memory for new items. + Evictions uint64 + // Reclaimed Number of times an entry was stored using memory from an expired entry. + Reclaimed uint64 + // BytesRead Total number of bytes read by this server from network. + BytesRead uint64 + // BytesWritten Total number of bytes sent by this server to network. + BytesWritten uint64 + // LimitMaxbytes Number of bytes this server is allowed to use for storage. + LimitMaxbytes uint64 + // ListenDisabledNum Number of times server has stopped accepting new connections (maxconns). + ListenDisabledNum uint64 + // TimeInListenDisabledUs Number of microseconds in maxconns. + TimeInListenDisabledUs uint64 + // ConnYields Number of times any connection yielded to another due to hitting the -R limit. + ConnYields uint64 + // HashBytes Bytes currently used by hash tables. + HashBytes uint64 + // ExpiredUnfetched Items pulled from LRU that were never touched by get/incr/append/etc before expiring. + ExpiredUnfetched uint64 + // EvictedUnfetched Items evicted from LRU that were never touched by get/incr/append/etc. + EvictedUnfetched uint64 + // EvictedActive Items evicted from LRU that had been hit recently but did not jump to top of LRU. + EvictedActive uint64 + // SlabsMoved Total slab pages moved. + SlabsMoved uint64 + // CrawlerReclaimed Total items freed by LRU Crawler. + CrawlerReclaimed uint64 + // CrawlerItemsChecked Total items examined by LRU Crawler. + CrawlerItemsChecked uint64 + // LrutailReflocked Times LRU tail was found with active ref. Items can be evicted to avoid OOM errors. + LrutailReflocked uint64 + // MovesToCold Items moved from HOT/WARM to COLD LRU's. + MovesToCold uint64 + // MovesToWarm Items moved from COLD to WARM LRU. + MovesToWarm uint64 + // MovesWithinLru Items reshuffled within HOT or WARM LRU's. + MovesWithinLru uint64 + // DirectReclaims Times worker threads had to directly reclaim or evict items. + DirectReclaims uint64 + // LruCrawlerStarts Times an LRU crawler was started. + LruCrawlerStarts uint64 + // LruMaintainerJuggles Number of times the LRU bg thread woke up. + LruMaintainerJuggles uint64 + // SlabReassignRescues Items rescued from eviction in page move. + SlabReassignRescues uint64 + // SlabReassignEvictionsNomem Valid items evicted during a page move (due to no free memory in slab). + SlabReassignEvictionsNomem uint64 + // SlabReassignChunkRescues Individual sections of an item rescued during a page move. + SlabReassignChunkRescues uint64 + // SlabReassignInlineReclaim Internal stat counter for when the page mover clears memory from the chunk freelist when it wasn't expecting to. + SlabReassignInlineReclaim uint64 + // SlabReassignBusyItems Items busy during page move, requiring a retry before page can be moved. + SlabReassignBusyItems uint64 + // SlabReassignBusyDeletes Items busy during page move, requiring deletion before page can be moved. + SlabReassignBusyDeletes uint64 + // LogWorkerDropped Logs a worker never wrote due to full buf. + LogWorkerDropped uint64 + // LogWorkerWritten Logs written by a worker, to be picked up. + LogWorkerWritten uint64 + // LogWatcherSkipped Logs not sent to slow watchers. + LogWatcherSkipped uint64 + // LogWatcherSent Logs written to watchers. + LogWatcherSent uint64 +} + +// StatsServers returns the general statistics of the servers +// retrieved with the `stats` command +func (c *Client) StatsServers() (servers map[net.Addr]*ServerStats, err error) { + // Each ServerStats has its own "asociated" sync.Mutex. + servers = make(map[net.Addr]*ServerStats) + muxes := make(map[net.Addr]*sync.Mutex) + + // addrs slice of the all the server adresses from the selector. + addrs := make([]net.Addr, 0) + + // For each server addres a ServerStats, sync.mutex is created. + err = c.selector.Each( + func(addr net.Addr) error { + addrs = append(addrs, addr) + servers[addr] = new(ServerStats) + muxes[addr] = new(sync.Mutex) + return nil + }, + ) + if err != nil { + return + } + + var wg sync.WaitGroup + wg.Add(len(addrs)) + for _, addr := range addrs { + go func(addr net.Addr) { + mux := muxes[addr] + server := servers[addr] + c.statsFromAddr(server, mux, addr) + wg.Done() + }(addr) + } + wg.Wait() + return +} + +func (c *Client) statsFromAddr(server *ServerStats, mux *sync.Mutex, addr net.Addr) { + var wg sync.WaitGroup + err := c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + if _, err := fmt.Fprintf(rw, "stats\r\n"); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + + // STAT \r\n + for { + line, err := rw.ReadBytes('\n') + if err != nil { + return err + } + if bytes.Equal(line, resultEnd) { + return nil + } + + wg.Add(1) + go func() { + server.parseStatValue(string(line), mux) + wg.Done() + }() + } + }) + wg.Wait() + server.ServerErr = err + return +} + +func (server *ServerStats) parseStatValue(line string, mux *sync.Mutex) { + var err error + tkns := strings.Split(line, " ") + v := tkns[2][:len(tkns[2])-2] + mux.Lock() + defer mux.Unlock() + switch tkns[1] { + case "version": + server.Version = v + case "accepting_conns": + server.AcceptingConns = v == "1" + case "hash_is_expanding": + server.HashIsExpanding = v == "1" + case "slab_reassign_running": + server.SlabReassignRunning = v == "1" + case "pid": + server.PID, err = getUtin32Val(v) + case "uptime": + server.Uptime, err = getUtin32Val(v) + case "time": + server.Time, err = getUtin32Val(v) + case "rusage_user": + server.RusageUser, err = getFloat64Val(v) + case "rusage_system": + server.RusageSystem, err = getFloat64Val(v) + case "max_connections": + server.MaxConnections, err = getUtin32Val(v) + case "curr_connections": + server.CurrConnections, err = getUtin32Val(v) + case "total_connections": + server.TotalConnections, err = getUtin32Val(v) + case "connection_structures": + server.ConnectionStructures, err = getUtin32Val(v) + case "reserved_fds": + server.ReservedFds, err = getUtin32Val(v) + case "threads": + server.Threads, err = getUtin32Val(v) + case "hash_power_level": + server.HashPowerLevel, err = getUtin32Val(v) + case "slab_global_page_pool": + server.SlabGlobalPagePool, err = getUtin32Val(v) + case "pointer_size": + server.PointerSize, err = getUint64Val(v) + case "curr_items": + server.CurrItems, err = getUint64Val(v) + case "total_items": + server.TotalItems, err = getUint64Val(v) + case "bytes": + server.Bytes, err = getUint64Val(v) + case "rejected_connections": + server.RejectedConnections, err = getUint64Val(v) + case "cmd_get": + server.CmdGet, err = getUint64Val(v) + case "cmd_set": + server.CmdSet, err = getUint64Val(v) + case "cmd_flush": + server.CmdFlush, err = getUint64Val(v) + case "cmd_touch": + server.CmdTouch, err = getUint64Val(v) + case "get_hits": + server.GetHits, err = getUint64Val(v) + case "get_misses": + server.GetMisses, err = getUint64Val(v) + case "get_expired": + server.GetExpired, err = getUint64Val(v) + case "get_flushed": + server.GetFlushed, err = getUint64Val(v) + case "delete_misses": + server.DeleteMisses, err = getUint64Val(v) + case "delete_hits": + server.DeleteHits, err = getUint64Val(v) + case "incr_misses": + server.IncrMisses, err = getUint64Val(v) + case "incr_hits": + server.IncrHits, err = getUint64Val(v) + case "decr_misses": + server.DecrMisses, err = getUint64Val(v) + case "decr_hits": + server.DecrHits, err = getUint64Val(v) + case "cas_misses": + server.CasMisses, err = getUint64Val(v) + case "cas_hits": + server.CasHits, err = getUint64Val(v) + case "cas_badval": + server.CasBadval, err = getUint64Val(v) + case "touch_hits": + server.TouchHits, err = getUint64Val(v) + case "touch_misses": + server.TouchMisses, err = getUint64Val(v) + case "auth_cmds": + server.AuthCmds, err = getUint64Val(v) + case "auth_errors": + server.AuthErrors, err = getUint64Val(v) + case "idle_kicks": + server.IdleKicks, err = getUint64Val(v) + case "evictions": + server.Evictions, err = getUint64Val(v) + case "reclaimed": + server.Reclaimed, err = getUint64Val(v) + case "bytes_read": + server.BytesRead, err = getUint64Val(v) + case "bytes_written": + server.BytesWritten, err = getUint64Val(v) + case "limit_maxbytes": + server.LimitMaxbytes, err = getUint64Val(v) + case "listen_disabled_num": + server.ListenDisabledNum, err = getUint64Val(v) + case "time_in_listen_disabled_us": + server.TimeInListenDisabledUs, err = getUint64Val(v) + case "conn_yields": + server.ConnYields, err = getUint64Val(v) + case "hash_bytes": + server.HashBytes, err = getUint64Val(v) + case "expired_unfetched": + server.ExpiredUnfetched, err = getUint64Val(v) + case "evicted_unfetched": + server.EvictedUnfetched, err = getUint64Val(v) + case "evicted_active": + server.EvictedActive, err = getUint64Val(v) + case "slabs_moved": + server.SlabsMoved, err = getUint64Val(v) + case "crawler_reclaimed": + server.CrawlerReclaimed, err = getUint64Val(v) + case "crawler_items_checked": + server.CrawlerItemsChecked, err = getUint64Val(v) + case "lrutail_reflocked": + server.LrutailReflocked, err = getUint64Val(v) + case "moves_to_cold": + server.MovesToCold, err = getUint64Val(v) + case "moves_to_warm": + server.MovesToWarm, err = getUint64Val(v) + case "moves_within_lru": + server.MovesWithinLru, err = getUint64Val(v) + case "direct_reclaims": + server.DirectReclaims, err = getUint64Val(v) + case "lru_crawler_starts": + server.LruCrawlerStarts, err = getUint64Val(v) + case "lru_maintainer_juggles": + server.LruMaintainerJuggles, err = getUint64Val(v) + case "slab_reassign_rescues": + server.SlabReassignRescues, err = getUint64Val(v) + case "slab_reassign_evictions_nomem": + server.SlabReassignEvictionsNomem, err = getUint64Val(v) + case "slab_reassign_chunk_rescues": + server.SlabReassignChunkRescues, err = getUint64Val(v) + case "slab_reassign_inline_reclaim": + server.SlabReassignInlineReclaim, err = getUint64Val(v) + case "slab_reassign_busy_items": + server.SlabReassignBusyItems, err = getUint64Val(v) + case "slab_reassign_busy_deletes": + server.SlabReassignBusyDeletes, err = getUint64Val(v) + case "log_worker_dropped": + server.LogWorkerDropped, err = getUint64Val(v) + case "log_worker_written": + server.LogWorkerWritten, err = getUint64Val(v) + case "log_watcher_skipped": + server.LogWatcherSkipped, err = getUint64Val(v) + case "log_watcher_sent": + server.LogWatcherSent, err = getUint64Val(v) + } + if err != nil { + err = errors.New("Unable to parse value for " + tkns[1] + " stat " + err.Error()) + server.Errs.AppendError(err) + } + +} + +func getUtin32Val(v string) (uint32, error) { + i, err := strconv.ParseUint(v, 10, 64) + return uint32(i), err +} +func getUint64Val(v string) (uint64, error) { + return strconv.ParseUint(v, 10, 64) +} + +func getFloat64Val(v string) (float64, error) { + return strconv.ParseFloat(v, 64) +} diff --git a/memcache/stats_items.go b/memcache/stats_items.go new file mode 100644 index 00000000..4d29ad7b --- /dev/null +++ b/memcache/stats_items.go @@ -0,0 +1,380 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memcache + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "net" + "regexp" + "strconv" + "strings" + "sync" +) + +var cachedumpItem = regexp.MustCompile("ITEM (.*) \\[(\\d+) b; (\\d+) s\\]") + +//DumpKey information of a stored key. +type DumpKey struct { + // Key The item key + Key string + // Size Item size (including key) in bytes + Size uint64 + // Expiration expiration time, as a unix timestamp. + Expiration uint64 +} + +// StatsItemsServer information about item storage per slab class. +type StatsItemsServer struct { + // ServerErr Error if can't get the stats information. + ServerErr error + // StatsItemsSlabs statistics for each slab. + StatsItemsSlabs map[string]*StatsItems + // DumpErr Error if can't cachedump + DumpErr error +} + +// StatsItems information of the items in the slab. +type StatsItems struct { + // Errs contains the errors that occurred while parsing the response + Errs errorsSlice + //Retrieved dumped keys + Keys []DumpKey + + // Number Number of items presently stored in this class. Expired items are not automatically excluded. + Number uint64 + // NumberHot Number of items presently stored in the HOT LRU. + NumberHot uint64 + // NumberWarm Number of items presently stored in the WARM LRU. + NumberWarm uint64 + // NumberCold Number of items presently stored in the COLD LRU. + NumberCold uint64 + // NumberTemp Number of items presently stored in the TEMPORARY LRU. + NumberTemp uint64 + // AgeHot Age of the oldest item in HOT LRU. + AgeHot uint64 + // AgeWarm Age of the oldest item in WARM LRU. + AgeWarm uint64 + // Age Age of the oldest item in the LRU. + Age uint64 + // Evicted Number of times an item had to be evicted from the LRU before it expired. + Evicted uint64 + // EvictedNonzero Number of times an item which had an explicit expire time set had to be evicted from the LRU before it expired. + EvictedNonzero uint64 + // EvictedTime Seconds since the last access for the most recent item evicted from this class. Use this to judge how recently active your evicted data is. + EvictedTime uint64 + // Outofmemory Number of times the underlying slab class was unable to store a new item. This means you are running with -M or an eviction failed. + Outofmemory uint64 + // Tailrepairs Number of times we self-healed a slab with a refcount leak. If this counter is increasing a lot, please report your situation to the developers. + Tailrepairs uint64 + // Reclaimed Number of times an entry was stored using memory from an expired entry. + Reclaimed uint64 + // ExpiredUnfetched Number of expired items reclaimed from the LRU which were never touched after being set. + ExpiredUnfetched uint64 + // EvictedUnfetched Number of valid items evicted from the LRU which were never touched after being set. + EvictedUnfetched uint64 + // EvictedActive Number of valid items evicted from the LRU which were recently touched but were evicted before being moved to the top of the LRU again. + EvictedActive uint64 + // CrawlerReclaimed Number of items freed by the LRU Crawler. + CrawlerReclaimed uint64 + // LrutailReflocked Number of items found to be refcount locked in the LRU tail. + LrutailReflocked uint64 + // MovesToCold Number of items moved from HOT or WARM into COLD. + MovesToCold uint64 + // MovesToWarm Number of items moved from COLD to WARM. + MovesToWarm uint64 + // MovesWithinLru Number of times active items were bumped within HOT or WARM. + MovesWithinLru uint64 + // DirectReclaims Number of times worker threads had to directly pull LRU tails to find memory for a new item. + DirectReclaims uint64 + // HitsToHot Number of keys that have been requested and found present in the HOT LRU. + HitsToHot uint64 + // HitsToWarm Number of keys that have been requested and found present in the WARM LRU. + HitsToWarm uint64 + // HitsToCold Number of keys that have been requested and found present in the COLD LRU. + HitsToCold uint64 + // HitsToTemp Number of keys that have been requested and found present in each sub-LRU. + HitsToTemp uint64 +} + +// StatsItemsServers returns information about item storage per slab class of all the servers, +// retrieved with the `stats items` command +// +// maxDumpKeys is the maximum number of keys that are going to be retrieved +// if maxDumpKeys < 0, doesn't dump the keys +func (c *Client) StatsItemsServers(maxDumpKeys int) (servers map[net.Addr]*StatsItemsServer, err error) { + // Each ServerStats has its own "asociated" sync.Mutex. + servers = make(map[net.Addr]*StatsItemsServer) + muxes := make(map[net.Addr]*sync.Mutex) + + // addrs slice of the all the server adresses from the selector. + addrs := make([]net.Addr, 0) + + // For each server addres a StatsItemsServer, sync.mutex is created. + err = c.selector.Each( + func(addr net.Addr) error { + addrs = append(addrs, addr) + servers[addr] = &StatsItemsServer{StatsItemsSlabs: make(map[string]*StatsItems)} + muxes[addr] = new(sync.Mutex) + return nil + }, + ) + if err != nil { + return + } + + var wg sync.WaitGroup + wg.Add(len(addrs)) + for _, addr := range addrs { + go func(addr net.Addr) { + mux := muxes[addr] + server := servers[addr] + c.statsItemsFromAddr(server, mux, addr) + wg.Done() + }(addr) + } + wg.Wait() + + wg.Add(len(addrs)) + if maxDumpKeys >= 0 { + for _, addr := range addrs { + go func(addr net.Addr) { + mux := muxes[addr] + server := servers[addr] + c.cachedumpFromAddr(server, maxDumpKeys, mux, addr) + wg.Done() + }(addr) + } + } + wg.Wait() + return +} + +// From the protocol definition for the command stats items. +// +// ------------------------------------------------------------------------ +// Item statistics. +// +// CAVEAT: This section describes statistics which are subject to change in the +// future. +// +// The "stats" command with the argument of "items" returns information about +// item storage per slab class. The data is returned in the format: +// +// STAT items:: \r\n +// +// The server terminates this list with the line. +// +// END\r\n +// +// The slabclass aligns with class ids used by the "stats slabs" command. Where +// "stats slabs" describes size and memory usage, "stats items" shows higher +// level information. +// +// The following item values are defined as of writing. +// ------------------------------------------------------------------------ +func (c *Client) statsItemsFromAddr(sis *StatsItemsServer, mux *sync.Mutex, addr net.Addr) { + var wg sync.WaitGroup + err := c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + if _, err := fmt.Fprintf(rw, "stats items\r\n"); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + + for { + line, err := rw.ReadBytes('\n') + if err != nil { + return err + } + if bytes.Equal(line, resultEnd) { + return nil + } + + wg.Add(1) + go func() { + // STAT items:: \r\n + sis.parseStat(string(line[:len(line)-2]), mux) + wg.Done() + }() + } + }) + wg.Wait() + sis.ServerErr = err + return +} + +func (sis *StatsItemsServer) parseStat(line string, mux *sync.Mutex) { + // STAT items:: + tkns := strings.Split(line, ":") + statValue := strings.Split(tkns[2], " ") + mux.Lock() + defer mux.Unlock() + + slab := sis.StatsItemsSlabs[tkns[1]] + if slab == nil { + slab = new(StatsItems) + sis.StatsItemsSlabs[tkns[1]] = slab + } + slab.setStatValue(statValue[0], statValue[1]) +} + +func (si *StatsItems) setStatValue(stat, value string) { + val, err := strconv.ParseUint(value, 10, 64) + if err != nil { + err = errors.New("Unable to parse value for " + stat + " stat " + err.Error()) + si.Errs.AppendError(err) + return + } + switch stat { + case "number": + si.Number = val + case "number_hot": + si.NumberHot = val + case "number_warm": + si.NumberWarm = val + case "number_cold": + si.NumberCold = val + case "number_temp": + si.NumberTemp = val + case "age_hot": + si.AgeHot = val + case "age_warm": + si.AgeWarm = val + case "age": + si.Age = val + case "evicted": + si.Evicted = val + case "evicted_nonzero": + si.EvictedNonzero = val + case "evicted_time": + si.EvictedTime = val + case "outofmemory": + si.Outofmemory = val + case "tailrepairs": + si.Tailrepairs = val + case "reclaimed": + si.Reclaimed = val + case "expired_unfetched": + si.ExpiredUnfetched = val + case "evicted_unfetched": + si.EvictedUnfetched = val + case "evicted_active": + si.EvictedActive = val + case "crawler_reclaimed": + si.CrawlerReclaimed = val + case "lrutail_reflocked": + si.LrutailReflocked = val + case "moves_to_cold": + si.MovesToCold = val + case "moves_to_warm": + si.MovesToWarm = val + case "moves_within_lru": + si.MovesWithinLru = val + case "direct_reclaims": + si.DirectReclaims = val + case "hits_to_hot": + si.HitsToHot = val + case "hits_to_warm": + si.HitsToWarm = val + case "hits_to_cold": + si.HitsToCold = val + case "hits_to_temp": + si.HitsToTemp = val + } +} + +func (c *Client) cachedumpFromAddr(sis *StatsItemsServer, maxDumpKeys int, mux *sync.Mutex, addr net.Addr) { + var wg sync.WaitGroup + l := len(sis.StatsItemsSlabs) + if l == 0 { + return + } + err := c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + line, err := writeReadLine(rw, "stats cachedump\r\n") + if err != nil { + return err + } + if bytes.Equal(line, []byte("CLIENT_ERROR stats cachedump not allowed\r\n")) { + return ErrDumpDisable + } + return nil + }) + if err != nil { + sis.DumpErr = err + } + + wg.Add(l) + for slab, stats := range sis.StatsItemsSlabs { + go func(slab string, stats *StatsItems) { + c.cachedumpSlabFromAddr(slab, stats, maxDumpKeys, mux, addr) + wg.Done() + }(slab, stats) + } + wg.Wait() + // + return +} + +func (c *Client) cachedumpSlabFromAddr(slab string, si *StatsItems, maxDumpKeys int, mux *sync.Mutex, addr net.Addr) (err error) { + var wg sync.WaitGroup + err = c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + if _, err := fmt.Fprintf(rw, "stats cachedump %s %d\r\n", slab, maxDumpKeys); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + + for { + line, err := rw.ReadBytes('\n') + if err != nil { + return err + } + if bytes.Equal(line, resultEnd) { + return nil + } + wg.Add(1) + go func() { + //ITEM [ b; s]\r\n + si.parseAddDumpKey(string(line), mux) + wg.Done() + }() + } + }) + wg.Wait() + return +} + +func (si *StatsItems) parseAddDumpKey(line string, mux *sync.Mutex) { + tkns := cachedumpItem.FindStringSubmatch(line) + if len(tkns) != 4 { + mux.Lock() + si.Errs.AppendError(fmt.Errorf("regex didn't match response correctly")) + mux.Unlock() + return + } + dk := DumpKey{Key: tkns[1]} + dk.Size, _ = strconv.ParseUint(tkns[2], 10, 64) + dk.Expiration, _ = strconv.ParseUint(tkns[3], 10, 64) + mux.Lock() + si.Keys = append(si.Keys, dk) + mux.Unlock() +} diff --git a/memcache/stats_slabs.go b/memcache/stats_slabs.go new file mode 100644 index 00000000..32db4d3c --- /dev/null +++ b/memcache/stats_slabs.go @@ -0,0 +1,244 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memcache + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" +) + +//StatsSlabsServer returns information,broken down by slab about items stored in memcached +// More centered to performance of a slab rather than counts of particular items. +type StatsSlabsServer struct { + // ServerErr Error if can't get the stats information. + ServerErr error + // StatsSlabs statistics for each slab. + StatsSlabs map[string]*StatsSlab + //ActiveSlabs Total number of slab classes allocated. + ActiveSlabs uint64 + //TotalMalloced Total amount of memory allocated to slab pages. + TotalMalloced uint64 +} + +//StatsSlab statistics for one slab ` +type StatsSlab struct { + // Errs contains the errors that occurred while parsing the response + Errs errorsSlice + // ChunkSize The amount of space each chunk uses. One item will use one chunk of the appropriate size. + ChunkSize uint64 + // ChunksPerPage How many chunks exist within one page. A page by default is less than or equal to one megabyte in size. Slabs are allocated by page, then broken into chunks. + ChunksPerPage uint64 + // TotalPages Total number of pages allocated to the slab class. + TotalPages uint64 + // TotalChunks Total number of chunks allocated to the slab class. + TotalChunks uint64 + // GetHits Total number of get requests serviced by this class. + GetHits uint64 + // CmdSet Total number of set requests storing data in this class. + CmdSet uint64 + // DeleteHits Total number of successful deletes from this class. + DeleteHits uint64 + // IncrHits Total number of incrs modifying this class. + IncrHits uint64 + // DecrHits Total number of decrs modifying this class. + DecrHits uint64 + // CasHits Total number of CAS commands modifying this class. + CasHits uint64 + // CasBadval Total number of CAS commands that failed to modify a value due to a bad CAS id. + CasBadval uint64 + // TouchHits Total number of touches serviced by this class. + TouchHits uint64 + // UsedChunks How many chunks have been allocated to items. + UsedChunks uint64 + // FreeChunks Chunks not yet allocated to items, or freed via delete. + FreeChunks uint64 + // FreeChunksEnd Number of free chunks at the end of the last allocated page. + FreeChunksEnd uint64 + // MemRequested Number of bytes requested to be stored in this slab[*]. + MemRequested uint64 + // ActiveSlabs Total number of slab classes allocated. + ActiveSlabs uint64 + // TotalMalloced Total amount of memory allocated to slab pages. + TotalMalloced uint64 +} + +//StatsSlabsServers returns information about the storage per slab class of the servers, +// retrieved with the `stats slabs` command +func (c *Client) StatsSlabsServers() (servers map[net.Addr]*StatsSlabsServer, err error) { + // Each ServerStats has its own "asociated" sync.Mutex. + servers = make(map[net.Addr]*StatsSlabsServer) + muxes := make(map[net.Addr]*sync.Mutex) + + // addrs slice of the all the server adresses from the selector. + addrs := make([]net.Addr, 0) + + // For each server addres a StatsSlabsServer, sync.mutex is created. + err = c.selector.Each( + func(addr net.Addr) error { + addrs = append(addrs, addr) + servers[addr] = &StatsSlabsServer{StatsSlabs: make(map[string]*StatsSlab)} + muxes[addr] = new(sync.Mutex) + return nil + }, + ) + if err != nil { + return + } + + var wg sync.WaitGroup + wg.Add(len(addrs)) + for _, addr := range addrs { + go func(addr net.Addr) { + mux := muxes[addr] + server := servers[addr] + c.statsSlabsFromAddr(server, mux, addr) + wg.Done() + }(addr) + } + wg.Wait() + return +} + +//From the protocol definition for the command stats items +// +//Slab statistics +//--------------- +//CAVEAT: This section describes statistics which are subject to change in the +//future. +// +//The "stats" command with the argument of "slabs" returns information about +//each of the slabs created by memcached during runtime. This includes per-slab +//information along with some totals. The data is returned in the format: +// +//STAT : \r\n +//STAT \r\n +// +//The server terminates this list with the line +// +//END\r\n +func (c *Client) statsSlabsFromAddr(sss *StatsSlabsServer, mux *sync.Mutex, addr net.Addr) (err error) { + var wg sync.WaitGroup + err = c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { + if _, err := fmt.Fprintf(rw, "stats slabs\r\n"); err != nil { + return err + } + if err := rw.Flush(); err != nil { + return err + } + + for { + line, err := rw.ReadBytes('\n') + if err != nil { + return err + } + if bytes.Equal(line, resultEnd) { + return err + } + //STAT : \r\n + //STAT \r\n + wg.Add(1) + go func() { + sss.parseStat(string(line[5:len(line)-2]), mux) + wg.Done() + }() + } + }) + wg.Wait() + sss.ServerErr = err + return +} + +func (sss *StatsSlabsServer) parseStat(line string, mux *sync.Mutex) { + // + tkns := strings.Split(string(line), ":") + if len(tkns) == 1 { + tkns := strings.Split(string(line), " ") + mux.Lock() + if tkns[0] == "active_slabs" { + sss.ActiveSlabs, _ = strconv.ParseUint(tkns[1], 10, 64) + } else { + sss.TotalMalloced, _ = strconv.ParseUint(tkns[1], 10, 64) + } + mux.Unlock() + return + } + + //: + + statValue := strings.Split(tkns[1], " ") + mux.Lock() + defer mux.Unlock() + slab := sss.StatsSlabs[tkns[0]] + if slab == nil { + slab = new(StatsSlab) + sss.StatsSlabs[tkns[0]] = slab + } + slab.setStatValue(statValue[0], statValue[1]) +} + +func (slab *StatsSlab) setStatValue(stat, value string) { + v, err := getUint64Val(value) + if err != nil { + err = errors.New("Unable to parse value for stat" + stat + " stat " + err.Error()) + slab.Errs.AppendError(err) + return + } + switch stat { + case "chunk_size": + slab.ChunkSize = v + case "chunks_per_page": + slab.ChunksPerPage = v + case "total_pages": + slab.TotalPages = v + case "total_chunks": + slab.TotalChunks = v + case "get_hits": + slab.GetHits = v + case "cmd_set": + slab.CmdSet = v + case "delete_hits": + slab.DeleteHits = v + case "incr_hits": + slab.IncrHits = v + case "decr_hits": + slab.DecrHits = v + case "cas_hits": + slab.CasHits = v + case "cas_badval": + slab.CasBadval = v + case "touch_hits": + slab.TouchHits = v + case "used_chunks": + slab.UsedChunks = v + case "free_chunks": + slab.FreeChunks = v + case "free_chunks_end": + slab.FreeChunksEnd = v + case "mem_requested": + slab.MemRequested = v + case "active_slabs": + slab.ActiveSlabs = v + case "total_malloced": + slab.TotalMalloced = v + } +} From 0e85618fe00448147d0454f6ef9fb8d563d24aad Mon Sep 17 00:00:00 2001 From: jtorz <11037563+jtorz@users.noreply.github.com> Date: Sun, 13 Oct 2019 02:00:24 -0500 Subject: [PATCH 2/2] removed unnecesary go routines, using reflection to set values, comments with format --- memcache/memcache.go | 7 +- memcache/stats.go | 444 ++++++++++++++++------------------------ memcache/stats_items.go | 222 +++++++------------- memcache/stats_slabs.go | 191 +++++++---------- 4 files changed, 320 insertions(+), 544 deletions(-) diff --git a/memcache/memcache.go b/memcache/memcache.go index 3c108ee8..a9db527c 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -105,6 +105,7 @@ func legalKey(key string) bool { var ( crlf = []byte("\r\n") space = []byte(" ") + colon = []byte(":") resultOK = []byte("OK\r\n") resultStored = []byte("STORED\r\n") resultNotStored = []byte("NOT_STORED\r\n") @@ -311,10 +312,10 @@ func (c *Client) onItem(item *Item, fn func(*Client, *bufio.ReadWriter, *Item) e return nil } -//FlushAll Invalidates all existing cache items. +// FlushAll Invalidates all existing cache items. // -//This command does not pause the server, as it returns immediately. -//It does not free up or flush memory at all, it just causes all items to expire. +// This command does not pause the server, as it returns immediately. +// It does not free up or flush memory at all, it just causes all items to expire. func (c *Client) FlushAll() error { return c.selector.Each(c.flushAllFromAddr) } diff --git a/memcache/stats.go b/memcache/stats.go index 4537be4f..c354fc12 100644 --- a/memcache/stats.go +++ b/memcache/stats.go @@ -1,11 +1,11 @@ /* -Copyright 2011 Google Inc. +Copyright 2019 gomemcache Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http:// www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -22,13 +22,17 @@ import ( "errors" "fmt" "net" + "reflect" "strconv" "strings" "sync" + "unicode" ) type errorsSlice []error +var errUnknownStat = errors.New("unknown stat") + func (errs errorsSlice) Error() string { switch len(errs) { case 0: @@ -66,181 +70,188 @@ func (errs *errorsSlice) AppendError(err error) { // ServerStats contains the general statistics from one server. type ServerStats struct { - //ServerErr Error if can't get the stats information + // ServerErr error if can't get the stats information. ServerErr error - // Errs contains the errors that occurred while parsing the response + // Errs contains the errors that occurred while parsing the response. Errs errorsSlice + // UnknownStats contains the stats that are not in the struct. This can be useful if the memcached developers add new stats in newer versions. + UnknownStats map[string]string - // Version Version string of this server. + // Version version string of this server. Version string - // AcceptingConns Whether or not server is accepting conns. + // AcceptingConns whether or not server is accepting conns. AcceptingConns bool - // HashIsExpanding Indicates if the hash table is being grown to a new size. + // HashIsExpanding indicates if the hash table is being grown to a new size. HashIsExpanding bool - // SlabReassignRunning If a slab page is being moved. + // SlabReassignRunning if a slab page is being moved. SlabReassignRunning bool - // PID Process id of this server process. - PID uint32 - // Uptime Number of secs since the server started. + // Pid process id of this server process. + Pid uint32 + // Uptime number of secs since the server started. Uptime uint32 // Time current UNIX time according to the server. Time uint32 - // RusageUser Accumulated user time for this process (seconds:microseconds). + // RusageUser accumulated user time for this process (seconds:microseconds). RusageUser float64 - // RusageSystem Accumulated system time for this process (seconds:microseconds). + // RusageSystem accumulated system time for this process (seconds:microseconds). RusageSystem float64 - // MaxConnections Max number of simultaneous connections. + // MaxConnections max number of simultaneous connections. MaxConnections uint32 - // CurrConnections Number of open connections. + // CurrConnections number of open connections. CurrConnections uint32 - // TotalConnections Total number of connections opened since the server started running. + // TotalConnections total number of connections opened since the server started running. TotalConnections uint32 - // ConnectionStructures Number of connection structures allocated by the server. + // ConnectionStructures number of connection structures allocated by the server. ConnectionStructures uint32 - // ReservedFds Number of misc fds used internally. + // ReservedFds number of misc fds used internally. ReservedFds uint32 - // Threads Number of worker threads requested. (see doc/threads.txt). + // Threads number of worker threads requested. (see doc/threads.txt). Threads uint32 - // HashPowerLevel Current size multiplier for hash table. + // HashPowerLevel current size multiplier for hash table. HashPowerLevel uint32 - // SlabGlobalPagePool Slab pages returned to global pool for reassignment to other slab classes. + // SlabGlobalPagePool slab pages returned to global pool for reassignment to other slab classes. SlabGlobalPagePool uint32 - // PointerSize Default size of pointers on the host OS (generally 32 or 64). + // PointerSize default size of pointers on the host OS (generally 32 or 64). PointerSize uint64 - // CurrItems Current number of items stored. + // CurrItems current number of items stored. CurrItems uint64 - // TotalItems Total number of items stored since the server started. + // TotalItems total number of items stored since the server started. TotalItems uint64 - // Bytes Current number of bytes used to store items. + // Bytes current number of bytes used to store items. Bytes uint64 - // RejectedConnections Conns rejected in maxconns_fast mode. + // RejectedConnections conns rejected in maxconns_fast mode. RejectedConnections uint64 - // CmdGet Cumulative number of retrieval reqs. + // CmdGet cumulative number of retrieval reqs. CmdGet uint64 - // CmdSet Cumulative number of storage reqs. + // CmdSet cumulative number of storage reqs. CmdSet uint64 - // CmdFlush Cumulative number of flush reqs. + // CmdFlush cumulative number of flush reqs. CmdFlush uint64 - // CmdTouch Cumulative number of touch reqs. + // CmdTouch cumulative number of touch reqs. CmdTouch uint64 - // GetHits Number of keys that have been requested and found present. + // GetHits number of keys that have been requested and found present. GetHits uint64 - // GetMisses Number of items that have been requested and not found. + // GetMisses number of items that have been requested and not found. GetMisses uint64 - // GetExpired Number of items that have been requested but had already expired. + // GetExpired number of items that have been requested but had already expired. GetExpired uint64 - // GetFlushed Number of items that have been requested but have been flushed via flush_all. + // GetFlushed number of items that have been requested but have been flushed via flush_all. GetFlushed uint64 - // DeleteMisses Number of deletions reqs for missing keys. + // DeleteMisses number of deletions reqs for missing keys. DeleteMisses uint64 - // DeleteHits Number of deletion reqs resulting in an item being removed. + // DeleteHits number of deletion reqs resulting in an item being removed. DeleteHits uint64 - // IncrMisses Number of incr reqs against missing keys. + // IncrMisses number of incr reqs against missing keys. IncrMisses uint64 - // IncrHits Number of successful incr reqs. + // IncrHits number of successful incr reqs. IncrHits uint64 - // DecrMisses Number of decr reqs against missing keys. + // DecrMisses number of decr reqs against missing keys. DecrMisses uint64 - // DecrHits Number of successful decr reqs. + // DecrHits number of successful decr reqs. DecrHits uint64 - // CasMisses Number of CAS reqs against missing keys. + // CasMisses number of CAS reqs against missing keys. CasMisses uint64 - // CasHits Number of successful CAS reqs. + // CasHits number of successful CAS reqs. CasHits uint64 - // CasBadval Number of CAS reqs for which a key was found, but the CAS value did not match. + // CasBadval number of CAS reqs for which a key was found, but the CAS value did not match. CasBadval uint64 - // TouchHits Number of keys that have been touched with a new expiration time. + // TouchHits number of keys that have been touched with a new expiration time. TouchHits uint64 - // TouchMisses Number of items that have been touched and not found. + // TouchMisses number of items that have been touched and not found. TouchMisses uint64 - // AuthCmds Number of authentication commands handled, success or failure. + // AuthCmds number of authentication commands handled, success or failure. AuthCmds uint64 - // AuthErrors Number of failed authentications. + // AuthErrors number of failed authentications. AuthErrors uint64 - // IdleKicks Number of connections closed due to reaching their idle timeout. + // IdleKicks number of connections closed due to reaching their idle timeout. IdleKicks uint64 - // Evictions Number of valid items removed from cache to free memory for new items. + // Evictions number of valid items removed from cache to free memory for new items. Evictions uint64 - // Reclaimed Number of times an entry was stored using memory from an expired entry. + // Reclaimed number of times an entry was stored using memory from an expired entry. Reclaimed uint64 - // BytesRead Total number of bytes read by this server from network. + // BytesRead total number of bytes read by this server from network. BytesRead uint64 - // BytesWritten Total number of bytes sent by this server to network. + // BytesWritten total number of bytes sent by this server to network. BytesWritten uint64 - // LimitMaxbytes Number of bytes this server is allowed to use for storage. + // LimitMaxbytes number of bytes this server is allowed to use for storage. LimitMaxbytes uint64 - // ListenDisabledNum Number of times server has stopped accepting new connections (maxconns). + // ListenDisabledNum number of times server has stopped accepting new connections (maxconns). ListenDisabledNum uint64 - // TimeInListenDisabledUs Number of microseconds in maxconns. + // TimeInListenDisabledUs number of microseconds in maxconns. TimeInListenDisabledUs uint64 - // ConnYields Number of times any connection yielded to another due to hitting the -R limit. + // ConnYields number of times any connection yielded to another due to hitting the -R limit. ConnYields uint64 - // HashBytes Bytes currently used by hash tables. + // HashBytes bytes currently used by hash tables. HashBytes uint64 - // ExpiredUnfetched Items pulled from LRU that were never touched by get/incr/append/etc before expiring. + // ExpiredUnfetched items pulled from LRU that were never touched by get/incr/append/etc before expiring. ExpiredUnfetched uint64 - // EvictedUnfetched Items evicted from LRU that were never touched by get/incr/append/etc. + // EvictedUnfetched items evicted from LRU that were never touched by get/incr/append/etc. EvictedUnfetched uint64 - // EvictedActive Items evicted from LRU that had been hit recently but did not jump to top of LRU. + // EvictedActive items evicted from LRU that had been hit recently but did not jump to top of LRU. EvictedActive uint64 - // SlabsMoved Total slab pages moved. + // SlabsMoved total slab pages moved. SlabsMoved uint64 - // CrawlerReclaimed Total items freed by LRU Crawler. + // CrawlerReclaimed total items freed by LRU Crawler. CrawlerReclaimed uint64 - // CrawlerItemsChecked Total items examined by LRU Crawler. + // CrawlerItemsChecked total items examined by LRU Crawler. CrawlerItemsChecked uint64 - // LrutailReflocked Times LRU tail was found with active ref. Items can be evicted to avoid OOM errors. + // LrutailReflocked times LRU tail was found with active ref. Items can be evicted to avoid OOM errors. LrutailReflocked uint64 - // MovesToCold Items moved from HOT/WARM to COLD LRU's. + // MovesToCold items moved from HOT/WARM to COLD LRU's. MovesToCold uint64 - // MovesToWarm Items moved from COLD to WARM LRU. + // MovesToWarm items moved from COLD to WARM LRU. MovesToWarm uint64 - // MovesWithinLru Items reshuffled within HOT or WARM LRU's. + // MovesWithinLru items reshuffled within HOT or WARM LRU's. MovesWithinLru uint64 - // DirectReclaims Times worker threads had to directly reclaim or evict items. + // DirectReclaims times worker threads had to directly reclaim or evict items. DirectReclaims uint64 - // LruCrawlerStarts Times an LRU crawler was started. + // LruCrawlerStarts times an LRU crawler was started. LruCrawlerStarts uint64 - // LruMaintainerJuggles Number of times the LRU bg thread woke up. + // LruMaintainerJuggles number of times the LRU bg thread woke up. LruMaintainerJuggles uint64 - // SlabReassignRescues Items rescued from eviction in page move. + // SlabReassignRescues items rescued from eviction in page move. SlabReassignRescues uint64 - // SlabReassignEvictionsNomem Valid items evicted during a page move (due to no free memory in slab). + // SlabReassignEvictionsNomem valid items evicted during a page move (due to no free memory in slab). SlabReassignEvictionsNomem uint64 - // SlabReassignChunkRescues Individual sections of an item rescued during a page move. + // SlabReassignChunkRescues individual sections of an item rescued during a page move. SlabReassignChunkRescues uint64 - // SlabReassignInlineReclaim Internal stat counter for when the page mover clears memory from the chunk freelist when it wasn't expecting to. + // SlabReassignInlineReclaim internal stat counter for when the page mover clears memory from the chunk freelist when it wasn't expecting to. SlabReassignInlineReclaim uint64 - // SlabReassignBusyItems Items busy during page move, requiring a retry before page can be moved. + // SlabReassignBusyItems items busy during page move, requiring a retry before page can be moved. SlabReassignBusyItems uint64 - // SlabReassignBusyDeletes Items busy during page move, requiring deletion before page can be moved. + // SlabReassignBusyDeletes items busy during page move, requiring deletion before page can be moved. SlabReassignBusyDeletes uint64 - // LogWorkerDropped Logs a worker never wrote due to full buf. + // LogWorkerDropped logs a worker never wrote due to full buf. LogWorkerDropped uint64 - // LogWorkerWritten Logs written by a worker, to be picked up. + // LogWorkerWritten logs written by a worker, to be picked up. LogWorkerWritten uint64 - // LogWatcherSkipped Logs not sent to slow watchers. + // LogWatcherSkipped logs not sent to slow watchers. LogWatcherSkipped uint64 - // LogWatcherSent Logs written to watchers. + // LogWatcherSent logs written to watchers. LogWatcherSent uint64 + // Libevent libevent string version. + Libevent string + // LruCrawlerRunning crawl in progress. + LruCrawlerRunning bool + // MallocFails number of malloc fails. + MallocFails uint64 + // LruBumpsDropped lru total bumps dropped. + LruBumpsDropped uint64 } // StatsServers returns the general statistics of the servers -// retrieved with the `stats` command +// retrieved with the `stats` command. func (c *Client) StatsServers() (servers map[net.Addr]*ServerStats, err error) { - // Each ServerStats has its own "asociated" sync.Mutex. servers = make(map[net.Addr]*ServerStats) - muxes := make(map[net.Addr]*sync.Mutex) // addrs slice of the all the server adresses from the selector. addrs := make([]net.Addr, 0) - // For each server addres a ServerStats, sync.mutex is created. err = c.selector.Each( func(addr net.Addr) error { addrs = append(addrs, addr) servers[addr] = new(ServerStats) - muxes[addr] = new(sync.Mutex) + servers[addr].UnknownStats = make(map[string]string) return nil }, ) @@ -252,9 +263,8 @@ func (c *Client) StatsServers() (servers map[net.Addr]*ServerStats, err error) { wg.Add(len(addrs)) for _, addr := range addrs { go func(addr net.Addr) { - mux := muxes[addr] server := servers[addr] - c.statsFromAddr(server, mux, addr) + c.statsFromAddr(server, addr) wg.Done() }(addr) } @@ -262,8 +272,7 @@ func (c *Client) StatsServers() (servers map[net.Addr]*ServerStats, err error) { return } -func (c *Client) statsFromAddr(server *ServerStats, mux *sync.Mutex, addr net.Addr) { - var wg sync.WaitGroup +func (c *Client) statsFromAddr(server *ServerStats, addr net.Addr) { err := c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { if _, err := fmt.Fprintf(rw, "stats\r\n"); err != nil { return err @@ -272,7 +281,6 @@ func (c *Client) statsFromAddr(server *ServerStats, mux *sync.Mutex, addr net.Ad return err } - // STAT \r\n for { line, err := rw.ReadBytes('\n') if err != nil { @@ -281,194 +289,90 @@ func (c *Client) statsFromAddr(server *ServerStats, mux *sync.Mutex, addr net.Ad if bytes.Equal(line, resultEnd) { return nil } - - wg.Add(1) - go func() { - server.parseStatValue(string(line), mux) - wg.Done() - }() + // STAT \r\n + tkns := bytes.Split(line[5:len(line)-2], space) + err = parseSetStatValue(reflect.ValueOf(server), string(tkns[0]), string(tkns[1])) + if err != nil { + if err != errUnknownStat { + server.Errs.AppendError(err) + } else { + server.UnknownStats[string(tkns[0])] = string(tkns[1]) + } + } } }) - wg.Wait() server.ServerErr = err return } -func (server *ServerStats) parseStatValue(line string, mux *sync.Mutex) { - var err error - tkns := strings.Split(line, " ") - v := tkns[2][:len(tkns[2])-2] - mux.Lock() - defer mux.Unlock() - switch tkns[1] { - case "version": - server.Version = v - case "accepting_conns": - server.AcceptingConns = v == "1" - case "hash_is_expanding": - server.HashIsExpanding = v == "1" - case "slab_reassign_running": - server.SlabReassignRunning = v == "1" - case "pid": - server.PID, err = getUtin32Val(v) - case "uptime": - server.Uptime, err = getUtin32Val(v) - case "time": - server.Time, err = getUtin32Val(v) - case "rusage_user": - server.RusageUser, err = getFloat64Val(v) - case "rusage_system": - server.RusageSystem, err = getFloat64Val(v) - case "max_connections": - server.MaxConnections, err = getUtin32Val(v) - case "curr_connections": - server.CurrConnections, err = getUtin32Val(v) - case "total_connections": - server.TotalConnections, err = getUtin32Val(v) - case "connection_structures": - server.ConnectionStructures, err = getUtin32Val(v) - case "reserved_fds": - server.ReservedFds, err = getUtin32Val(v) - case "threads": - server.Threads, err = getUtin32Val(v) - case "hash_power_level": - server.HashPowerLevel, err = getUtin32Val(v) - case "slab_global_page_pool": - server.SlabGlobalPagePool, err = getUtin32Val(v) - case "pointer_size": - server.PointerSize, err = getUint64Val(v) - case "curr_items": - server.CurrItems, err = getUint64Val(v) - case "total_items": - server.TotalItems, err = getUint64Val(v) - case "bytes": - server.Bytes, err = getUint64Val(v) - case "rejected_connections": - server.RejectedConnections, err = getUint64Val(v) - case "cmd_get": - server.CmdGet, err = getUint64Val(v) - case "cmd_set": - server.CmdSet, err = getUint64Val(v) - case "cmd_flush": - server.CmdFlush, err = getUint64Val(v) - case "cmd_touch": - server.CmdTouch, err = getUint64Val(v) - case "get_hits": - server.GetHits, err = getUint64Val(v) - case "get_misses": - server.GetMisses, err = getUint64Val(v) - case "get_expired": - server.GetExpired, err = getUint64Val(v) - case "get_flushed": - server.GetFlushed, err = getUint64Val(v) - case "delete_misses": - server.DeleteMisses, err = getUint64Val(v) - case "delete_hits": - server.DeleteHits, err = getUint64Val(v) - case "incr_misses": - server.IncrMisses, err = getUint64Val(v) - case "incr_hits": - server.IncrHits, err = getUint64Val(v) - case "decr_misses": - server.DecrMisses, err = getUint64Val(v) - case "decr_hits": - server.DecrHits, err = getUint64Val(v) - case "cas_misses": - server.CasMisses, err = getUint64Val(v) - case "cas_hits": - server.CasHits, err = getUint64Val(v) - case "cas_badval": - server.CasBadval, err = getUint64Val(v) - case "touch_hits": - server.TouchHits, err = getUint64Val(v) - case "touch_misses": - server.TouchMisses, err = getUint64Val(v) - case "auth_cmds": - server.AuthCmds, err = getUint64Val(v) - case "auth_errors": - server.AuthErrors, err = getUint64Val(v) - case "idle_kicks": - server.IdleKicks, err = getUint64Val(v) - case "evictions": - server.Evictions, err = getUint64Val(v) - case "reclaimed": - server.Reclaimed, err = getUint64Val(v) - case "bytes_read": - server.BytesRead, err = getUint64Val(v) - case "bytes_written": - server.BytesWritten, err = getUint64Val(v) - case "limit_maxbytes": - server.LimitMaxbytes, err = getUint64Val(v) - case "listen_disabled_num": - server.ListenDisabledNum, err = getUint64Val(v) - case "time_in_listen_disabled_us": - server.TimeInListenDisabledUs, err = getUint64Val(v) - case "conn_yields": - server.ConnYields, err = getUint64Val(v) - case "hash_bytes": - server.HashBytes, err = getUint64Val(v) - case "expired_unfetched": - server.ExpiredUnfetched, err = getUint64Val(v) - case "evicted_unfetched": - server.EvictedUnfetched, err = getUint64Val(v) - case "evicted_active": - server.EvictedActive, err = getUint64Val(v) - case "slabs_moved": - server.SlabsMoved, err = getUint64Val(v) - case "crawler_reclaimed": - server.CrawlerReclaimed, err = getUint64Val(v) - case "crawler_items_checked": - server.CrawlerItemsChecked, err = getUint64Val(v) - case "lrutail_reflocked": - server.LrutailReflocked, err = getUint64Val(v) - case "moves_to_cold": - server.MovesToCold, err = getUint64Val(v) - case "moves_to_warm": - server.MovesToWarm, err = getUint64Val(v) - case "moves_within_lru": - server.MovesWithinLru, err = getUint64Val(v) - case "direct_reclaims": - server.DirectReclaims, err = getUint64Val(v) - case "lru_crawler_starts": - server.LruCrawlerStarts, err = getUint64Val(v) - case "lru_maintainer_juggles": - server.LruMaintainerJuggles, err = getUint64Val(v) - case "slab_reassign_rescues": - server.SlabReassignRescues, err = getUint64Val(v) - case "slab_reassign_evictions_nomem": - server.SlabReassignEvictionsNomem, err = getUint64Val(v) - case "slab_reassign_chunk_rescues": - server.SlabReassignChunkRescues, err = getUint64Val(v) - case "slab_reassign_inline_reclaim": - server.SlabReassignInlineReclaim, err = getUint64Val(v) - case "slab_reassign_busy_items": - server.SlabReassignBusyItems, err = getUint64Val(v) - case "slab_reassign_busy_deletes": - server.SlabReassignBusyDeletes, err = getUint64Val(v) - case "log_worker_dropped": - server.LogWorkerDropped, err = getUint64Val(v) - case "log_worker_written": - server.LogWorkerWritten, err = getUint64Val(v) - case "log_watcher_skipped": - server.LogWatcherSkipped, err = getUint64Val(v) - case "log_watcher_sent": - server.LogWatcherSent, err = getUint64Val(v) - } - if err != nil { - err = errors.New("Unable to parse value for " + tkns[1] + " stat " + err.Error()) - server.Errs.AppendError(err) +// parseSetStatValue parses and sets the value of the stat to the corresponding struct field. +// +// In order to know the corresponding field of the stat, the snake_case stat name is converted to CamelCase +func parseSetStatValue(strPntr reflect.Value, stat, value string) error { + statCamel := toCamel(stat) + f := reflect.Indirect(strPntr).FieldByName(statCamel) + switch f.Kind() { + case reflect.Uint32: + uintv, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return errors.New("unable to parse uint value " + stat + ":" + err.Error()) + } + f.SetUint(uintv) + case reflect.Uint64: + uintv, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return errors.New("unable to parse uint value " + stat + ":" + err.Error()) + } + f.SetUint(uintv) + case reflect.Float64: + floatv, err := strconv.ParseFloat(value, 64) + if err != nil { + return errors.New("unable to parse float value for " + stat + ":" + err.Error()) + } + f.SetFloat(floatv) + case reflect.String: + f.SetString(value) + case reflect.Bool: + f.SetBool(value == "1") + default: + return errUnknownStat } - + return nil } -func getUtin32Val(v string) (uint32, error) { - i, err := strconv.ParseUint(v, 10, 64) - return uint32(i), err -} -func getUint64Val(v string) (uint64, error) { - return strconv.ParseUint(v, 10, 64) -} +func toCamel(s string) string { + if s == "" { + return "" + } + // Compute number of replacements. + m := strings.Count(s, "_") + if m == 0 { + return string(unicode.ToUpper(rune(s[0]))) + s[1:] // avoid allocation + } -func getFloat64Val(v string) (float64, error) { - return strconv.ParseFloat(v, 64) + // Apply replacements to buffer. + l := len(s) - m + if l == 0 { + return "" + } + t := make([]byte, l) + + w := 0 + start := 0 + for i := 0; i < m; i++ { + j := start + j += strings.Index(s[start:], "_") + if start != j { + t[w] = byte(unicode.ToUpper(rune(s[start]))) + w++ + w += copy(t[w:], s[start+1:j]) + } + start = j + 1 + } + if s[start:] != "" { + t[w] = byte(unicode.ToUpper(rune(s[start]))) + w++ + w += copy(t[w:], s[start+1:]) + } + return string(t[0:w]) } diff --git a/memcache/stats_items.go b/memcache/stats_items.go index 4d29ad7b..1603dc8a 100644 --- a/memcache/stats_items.go +++ b/memcache/stats_items.go @@ -1,5 +1,5 @@ /* -Copyright 2011 Google Inc. +Copyright 2019 gomemcache Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,19 +22,19 @@ import ( "errors" "fmt" "net" + "reflect" "regexp" "strconv" - "strings" "sync" ) var cachedumpItem = regexp.MustCompile("ITEM (.*) \\[(\\d+) b; (\\d+) s\\]") -//DumpKey information of a stored key. +// DumpKey information of a stored key. type DumpKey struct { - // Key The item key + // Key item key. Key string - // Size Item size (including key) in bytes + // Size item size (including key) in bytes. Size uint64 // Expiration expiration time, as a unix timestamp. Expiration uint64 @@ -42,96 +42,94 @@ type DumpKey struct { // StatsItemsServer information about item storage per slab class. type StatsItemsServer struct { - // ServerErr Error if can't get the stats information. + // ServerErr irror if can't get the stats information. ServerErr error // StatsItemsSlabs statistics for each slab. StatsItemsSlabs map[string]*StatsItems - // DumpErr Error if can't cachedump + // DumpErr error if can't cachedump. DumpErr error } // StatsItems information of the items in the slab. type StatsItems struct { - // Errs contains the errors that occurred while parsing the response + // Errs contains the errors that occurred while parsing the response. Errs errorsSlice - //Retrieved dumped keys + // Keys retrieved dumped keys. Keys []DumpKey + // UnknownStats contains the stats that are not in the struct. This can be useful if the memcached developers add new stats in newer versions. + UnknownStats map[string]string - // Number Number of items presently stored in this class. Expired items are not automatically excluded. + // Number number of items presently stored in this class. Expired items are not automatically excluded. Number uint64 - // NumberHot Number of items presently stored in the HOT LRU. + // NumberHot number of items presently stored in the HOT LRU. NumberHot uint64 - // NumberWarm Number of items presently stored in the WARM LRU. + // NumberWarm number of items presently stored in the WARM LRU. NumberWarm uint64 - // NumberCold Number of items presently stored in the COLD LRU. + // NumberCold number of items presently stored in the COLD LRU. NumberCold uint64 - // NumberTemp Number of items presently stored in the TEMPORARY LRU. + // NumberTemp number of items presently stored in the TEMPORARY LRU. NumberTemp uint64 - // AgeHot Age of the oldest item in HOT LRU. + // AgeHot age of the oldest item in HOT LRU. AgeHot uint64 - // AgeWarm Age of the oldest item in WARM LRU. + // AgeWarm age of the oldest item in WARM LRU. AgeWarm uint64 - // Age Age of the oldest item in the LRU. + // Age age of the oldest item in the LRU. Age uint64 - // Evicted Number of times an item had to be evicted from the LRU before it expired. + // Evicted number of times an item had to be evicted from the LRU before it expired. Evicted uint64 - // EvictedNonzero Number of times an item which had an explicit expire time set had to be evicted from the LRU before it expired. + // EvictedNonzero number of times an item which had an explicit expire time set had to be evicted from the LRU before it expired. EvictedNonzero uint64 - // EvictedTime Seconds since the last access for the most recent item evicted from this class. Use this to judge how recently active your evicted data is. + // EvictedTime seconds since the last access for the most recent item evicted from this class. Use this to judge how recently active your evicted data is. EvictedTime uint64 - // Outofmemory Number of times the underlying slab class was unable to store a new item. This means you are running with -M or an eviction failed. + // Outofmemory number of times the underlying slab class was unable to store a new item. This means you are running with -M or an eviction failed. Outofmemory uint64 - // Tailrepairs Number of times we self-healed a slab with a refcount leak. If this counter is increasing a lot, please report your situation to the developers. + // Tailrepairs number of times we self-healed a slab with a refcount leak. If this counter is increasing a lot, please report your situation to the developers. Tailrepairs uint64 - // Reclaimed Number of times an entry was stored using memory from an expired entry. + // Reclaimed number of times an entry was stored using memory from an expired entry. Reclaimed uint64 - // ExpiredUnfetched Number of expired items reclaimed from the LRU which were never touched after being set. + // ExpiredUnfetched number of expired items reclaimed from the LRU which were never touched after being set. ExpiredUnfetched uint64 - // EvictedUnfetched Number of valid items evicted from the LRU which were never touched after being set. + // EvictedUnfetched number of valid items evicted from the LRU which were never touched after being set. EvictedUnfetched uint64 - // EvictedActive Number of valid items evicted from the LRU which were recently touched but were evicted before being moved to the top of the LRU again. + // EvictedActive number of valid items evicted from the LRU which were recently touched but were evicted before being moved to the top of the LRU again. EvictedActive uint64 - // CrawlerReclaimed Number of items freed by the LRU Crawler. + // CrawlerReclaimed number of items freed by the LRU Crawler. CrawlerReclaimed uint64 - // LrutailReflocked Number of items found to be refcount locked in the LRU tail. + // LrutailReflocked number of items found to be refcount locked in the LRU tail. LrutailReflocked uint64 - // MovesToCold Number of items moved from HOT or WARM into COLD. + // MovesToCold number of items moved from HOT or WARM into COLD. MovesToCold uint64 - // MovesToWarm Number of items moved from COLD to WARM. + // MovesToWarm number of items moved from COLD to WARM. MovesToWarm uint64 - // MovesWithinLru Number of times active items were bumped within HOT or WARM. + // MovesWithinLru number of times active items were bumped within HOT or WARM. MovesWithinLru uint64 - // DirectReclaims Number of times worker threads had to directly pull LRU tails to find memory for a new item. + // DirectReclaims number of times worker threads had to directly pull LRU tails to find memory for a new item. DirectReclaims uint64 - // HitsToHot Number of keys that have been requested and found present in the HOT LRU. + // HitsToHot number of keys that have been requested and found present in the HOT LRU. HitsToHot uint64 - // HitsToWarm Number of keys that have been requested and found present in the WARM LRU. + // HitsToWarm number of keys that have been requested and found present in the WARM LRU. HitsToWarm uint64 - // HitsToCold Number of keys that have been requested and found present in the COLD LRU. + // HitsToCold number of keys that have been requested and found present in the COLD LRU. HitsToCold uint64 - // HitsToTemp Number of keys that have been requested and found present in each sub-LRU. + // HitsToTemp number of keys that have been requested and found present in each sub-LRU. HitsToTemp uint64 } // StatsItemsServers returns information about item storage per slab class of all the servers, -// retrieved with the `stats items` command +// retrieved with the `stats items` command. // // maxDumpKeys is the maximum number of keys that are going to be retrieved -// if maxDumpKeys < 0, doesn't dump the keys +// if maxDumpKeys < 0, doesn't dump the keys. func (c *Client) StatsItemsServers(maxDumpKeys int) (servers map[net.Addr]*StatsItemsServer, err error) { - // Each ServerStats has its own "asociated" sync.Mutex. servers = make(map[net.Addr]*StatsItemsServer) - muxes := make(map[net.Addr]*sync.Mutex) // addrs slice of the all the server adresses from the selector. addrs := make([]net.Addr, 0) - // For each server addres a StatsItemsServer, sync.mutex is created. err = c.selector.Each( func(addr net.Addr) error { addrs = append(addrs, addr) servers[addr] = &StatsItemsServer{StatsItemsSlabs: make(map[string]*StatsItems)} - muxes[addr] = new(sync.Mutex) return nil }, ) @@ -143,30 +141,28 @@ func (c *Client) StatsItemsServers(maxDumpKeys int) (servers map[net.Addr]*Stats wg.Add(len(addrs)) for _, addr := range addrs { go func(addr net.Addr) { - mux := muxes[addr] server := servers[addr] - c.statsItemsFromAddr(server, mux, addr) + c.statsItemsFromAddr(server, addr) wg.Done() }(addr) } wg.Wait() - wg.Add(len(addrs)) if maxDumpKeys >= 0 { + wg.Add(len(addrs)) for _, addr := range addrs { go func(addr net.Addr) { - mux := muxes[addr] server := servers[addr] - c.cachedumpFromAddr(server, maxDumpKeys, mux, addr) + c.cachedumpFromAddr(server, maxDumpKeys, addr) wg.Done() }(addr) } + wg.Wait() } - wg.Wait() return } -// From the protocol definition for the command stats items. +// From the protocol definition for the command stats items: // // ------------------------------------------------------------------------ // Item statistics. @@ -179,7 +175,7 @@ func (c *Client) StatsItemsServers(maxDumpKeys int) (servers map[net.Addr]*Stats // // STAT items:: \r\n // -// The server terminates this list with the line. +// The server terminates this list with the line: // // END\r\n // @@ -189,8 +185,7 @@ func (c *Client) StatsItemsServers(maxDumpKeys int) (servers map[net.Addr]*Stats // // The following item values are defined as of writing. // ------------------------------------------------------------------------ -func (c *Client) statsItemsFromAddr(sis *StatsItemsServer, mux *sync.Mutex, addr net.Addr) { - var wg sync.WaitGroup +func (c *Client) statsItemsFromAddr(sis *StatsItemsServer, addr net.Addr) { err := c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { if _, err := fmt.Fprintf(rw, "stats items\r\n"); err != nil { return err @@ -208,100 +203,37 @@ func (c *Client) statsItemsFromAddr(sis *StatsItemsServer, mux *sync.Mutex, addr return nil } - wg.Add(1) - go func() { - // STAT items:: \r\n - sis.parseStat(string(line[:len(line)-2]), mux) - wg.Done() - }() + // STAT items:: + sis.parseStat(line[11 : len(line)-2]) } }) - wg.Wait() sis.ServerErr = err return } -func (sis *StatsItemsServer) parseStat(line string, mux *sync.Mutex) { - // STAT items:: - tkns := strings.Split(line, ":") - statValue := strings.Split(tkns[2], " ") - mux.Lock() - defer mux.Unlock() - - slab := sis.StatsItemsSlabs[tkns[1]] +func (sis *StatsItemsServer) parseStat(line []byte) { + // : + tkns := bytes.FieldsFunc(line, func(r rune) bool { + return r == ':' || r == ' ' + }) + slabclass := string(tkns[0]) + slab := sis.StatsItemsSlabs[slabclass] if slab == nil { slab = new(StatsItems) - sis.StatsItemsSlabs[tkns[1]] = slab + slab.UnknownStats = make(map[string]string) + sis.StatsItemsSlabs[slabclass] = slab } - slab.setStatValue(statValue[0], statValue[1]) -} - -func (si *StatsItems) setStatValue(stat, value string) { - val, err := strconv.ParseUint(value, 10, 64) + err := parseSetStatValue(reflect.ValueOf(slab), string(tkns[1]), string(tkns[2])) if err != nil { - err = errors.New("Unable to parse value for " + stat + " stat " + err.Error()) - si.Errs.AppendError(err) - return - } - switch stat { - case "number": - si.Number = val - case "number_hot": - si.NumberHot = val - case "number_warm": - si.NumberWarm = val - case "number_cold": - si.NumberCold = val - case "number_temp": - si.NumberTemp = val - case "age_hot": - si.AgeHot = val - case "age_warm": - si.AgeWarm = val - case "age": - si.Age = val - case "evicted": - si.Evicted = val - case "evicted_nonzero": - si.EvictedNonzero = val - case "evicted_time": - si.EvictedTime = val - case "outofmemory": - si.Outofmemory = val - case "tailrepairs": - si.Tailrepairs = val - case "reclaimed": - si.Reclaimed = val - case "expired_unfetched": - si.ExpiredUnfetched = val - case "evicted_unfetched": - si.EvictedUnfetched = val - case "evicted_active": - si.EvictedActive = val - case "crawler_reclaimed": - si.CrawlerReclaimed = val - case "lrutail_reflocked": - si.LrutailReflocked = val - case "moves_to_cold": - si.MovesToCold = val - case "moves_to_warm": - si.MovesToWarm = val - case "moves_within_lru": - si.MovesWithinLru = val - case "direct_reclaims": - si.DirectReclaims = val - case "hits_to_hot": - si.HitsToHot = val - case "hits_to_warm": - si.HitsToWarm = val - case "hits_to_cold": - si.HitsToCold = val - case "hits_to_temp": - si.HitsToTemp = val + if err != errUnknownStat { + slab.Errs.AppendError(err) + } else { + slab.UnknownStats[string(tkns[1])] = string(tkns[2]) + } } } -func (c *Client) cachedumpFromAddr(sis *StatsItemsServer, maxDumpKeys int, mux *sync.Mutex, addr net.Addr) { +func (c *Client) cachedumpFromAddr(sis *StatsItemsServer, maxDumpKeys int, addr net.Addr) { var wg sync.WaitGroup l := len(sis.StatsItemsSlabs) if l == 0 { @@ -319,22 +251,21 @@ func (c *Client) cachedumpFromAddr(sis *StatsItemsServer, maxDumpKeys int, mux * }) if err != nil { sis.DumpErr = err + return } wg.Add(l) for slab, stats := range sis.StatsItemsSlabs { go func(slab string, stats *StatsItems) { - c.cachedumpSlabFromAddr(slab, stats, maxDumpKeys, mux, addr) + c.cachedumpSlabFromAddr(slab, stats, maxDumpKeys, addr) wg.Done() }(slab, stats) } wg.Wait() - // return } -func (c *Client) cachedumpSlabFromAddr(slab string, si *StatsItems, maxDumpKeys int, mux *sync.Mutex, addr net.Addr) (err error) { - var wg sync.WaitGroup +func (c *Client) cachedumpSlabFromAddr(slab string, si *StatsItems, maxDumpKeys int, addr net.Addr) (err error) { err = c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { if _, err := fmt.Fprintf(rw, "stats cachedump %s %d\r\n", slab, maxDumpKeys); err != nil { return err @@ -351,30 +282,21 @@ func (c *Client) cachedumpSlabFromAddr(slab string, si *StatsItems, maxDumpKeys if bytes.Equal(line, resultEnd) { return nil } - wg.Add(1) - go func() { - //ITEM [ b; s]\r\n - si.parseAddDumpKey(string(line), mux) - wg.Done() - }() + // ITEM [ b; s]\r\n + si.parseAddDumpKey(string(line)) } }) - wg.Wait() return } -func (si *StatsItems) parseAddDumpKey(line string, mux *sync.Mutex) { +func (si *StatsItems) parseAddDumpKey(line string) { tkns := cachedumpItem.FindStringSubmatch(line) if len(tkns) != 4 { - mux.Lock() - si.Errs.AppendError(fmt.Errorf("regex didn't match response correctly")) - mux.Unlock() + si.Errs.AppendError(errors.New("regex didn't match response correctly")) return } dk := DumpKey{Key: tkns[1]} dk.Size, _ = strconv.ParseUint(tkns[2], 10, 64) dk.Expiration, _ = strconv.ParseUint(tkns[3], 10, 64) - mux.Lock() si.Keys = append(si.Keys, dk) - mux.Unlock() } diff --git a/memcache/stats_slabs.go b/memcache/stats_slabs.go index 32db4d3c..4511fe22 100644 --- a/memcache/stats_slabs.go +++ b/memcache/stats_slabs.go @@ -1,5 +1,5 @@ /* -Copyright 2011 Google Inc. +Copyright 2019 gomemcache Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,85 +19,83 @@ package memcache import ( "bufio" "bytes" - "errors" "fmt" "net" + "reflect" "strconv" - "strings" "sync" ) -//StatsSlabsServer returns information,broken down by slab about items stored in memcached -// More centered to performance of a slab rather than counts of particular items. +// StatsSlabsServer information broken down by slab about items stored in memcached, +// more centered to performance of a slab rather than counts of particular items. type StatsSlabsServer struct { - // ServerErr Error if can't get the stats information. + // ServerErr error if can't get the stats information. ServerErr error // StatsSlabs statistics for each slab. StatsSlabs map[string]*StatsSlab - //ActiveSlabs Total number of slab classes allocated. + // ActiveSlabs total number of slab classes allocated. ActiveSlabs uint64 - //TotalMalloced Total amount of memory allocated to slab pages. + // TotalMalloced total amount of memory allocated to slab pages. TotalMalloced uint64 } -//StatsSlab statistics for one slab ` +// StatsSlab statistics for one slab. type StatsSlab struct { - // Errs contains the errors that occurred while parsing the response + // Errs contains the errors that occurred while parsing the response. Errs errorsSlice - // ChunkSize The amount of space each chunk uses. One item will use one chunk of the appropriate size. + // UnknownStats contains the stats that are not in the struct. This can be useful if the memcached developers add new stats in newer versions. + UnknownStats map[string]string + + // ChunkSize the amount of space each chunk uses. One item will use one chunk of the appropriate size. ChunkSize uint64 - // ChunksPerPage How many chunks exist within one page. A page by default is less than or equal to one megabyte in size. Slabs are allocated by page, then broken into chunks. + // ChunksPerPage how many chunks exist within one page. A page by default is less than or equal to one megabyte in size. Slabs are allocated by page, then broken into chunks. ChunksPerPage uint64 - // TotalPages Total number of pages allocated to the slab class. + // TotalPages total number of pages allocated to the slab class. TotalPages uint64 - // TotalChunks Total number of chunks allocated to the slab class. + // TotalChunks total number of chunks allocated to the slab class. TotalChunks uint64 - // GetHits Total number of get requests serviced by this class. + // GetHits total number of get requests serviced by this class. GetHits uint64 - // CmdSet Total number of set requests storing data in this class. + // CmdSet total number of set requests storing data in this class. CmdSet uint64 - // DeleteHits Total number of successful deletes from this class. + // DeleteHits total number of successful deletes from this class. DeleteHits uint64 - // IncrHits Total number of incrs modifying this class. + // IncrHits total number of incrs modifying this class. IncrHits uint64 - // DecrHits Total number of decrs modifying this class. + // DecrHits total number of decrs modifying this class. DecrHits uint64 - // CasHits Total number of CAS commands modifying this class. + // CasHits total number of CAS commands modifying this class. CasHits uint64 - // CasBadval Total number of CAS commands that failed to modify a value due to a bad CAS id. + // CasBadval total number of CAS commands that failed to modify a value due to a bad CAS id. CasBadval uint64 - // TouchHits Total number of touches serviced by this class. + // TouchHits total number of touches serviced by this class. TouchHits uint64 - // UsedChunks How many chunks have been allocated to items. + // UsedChunks how many chunks have been allocated to items. UsedChunks uint64 - // FreeChunks Chunks not yet allocated to items, or freed via delete. + // FreeChunks chunks not yet allocated to items, or freed via delete. FreeChunks uint64 - // FreeChunksEnd Number of free chunks at the end of the last allocated page. + // FreeChunksEnd number of free chunks at the end of the last allocated page. FreeChunksEnd uint64 - // MemRequested Number of bytes requested to be stored in this slab[*]. + // MemRequested number of bytes requested to be stored in this slab[*]. MemRequested uint64 - // ActiveSlabs Total number of slab classes allocated. + // ActiveSlabs total number of slab classes allocated. ActiveSlabs uint64 - // TotalMalloced Total amount of memory allocated to slab pages. + // TotalMalloced total amount of memory allocated to slab pages. TotalMalloced uint64 } -//StatsSlabsServers returns information about the storage per slab class of the servers, -// retrieved with the `stats slabs` command +// StatsSlabsServers returns information about the storage per slab class of the servers, +// retrieved with the `stats slabs` command. func (c *Client) StatsSlabsServers() (servers map[net.Addr]*StatsSlabsServer, err error) { - // Each ServerStats has its own "asociated" sync.Mutex. servers = make(map[net.Addr]*StatsSlabsServer) - muxes := make(map[net.Addr]*sync.Mutex) // addrs slice of the all the server adresses from the selector. addrs := make([]net.Addr, 0) - // For each server addres a StatsSlabsServer, sync.mutex is created. err = c.selector.Each( func(addr net.Addr) error { addrs = append(addrs, addr) servers[addr] = &StatsSlabsServer{StatsSlabs: make(map[string]*StatsSlab)} - muxes[addr] = new(sync.Mutex) return nil }, ) @@ -109,9 +107,8 @@ func (c *Client) StatsSlabsServers() (servers map[net.Addr]*StatsSlabsServer, er wg.Add(len(addrs)) for _, addr := range addrs { go func(addr net.Addr) { - mux := muxes[addr] server := servers[addr] - c.statsSlabsFromAddr(server, mux, addr) + c.statsSlabsFromAddr(server, addr) wg.Done() }(addr) } @@ -119,25 +116,24 @@ func (c *Client) StatsSlabsServers() (servers map[net.Addr]*StatsSlabsServer, er return } -//From the protocol definition for the command stats items +// From the protocol definition for the command stats items: // -//Slab statistics -//--------------- -//CAVEAT: This section describes statistics which are subject to change in the -//future. +// Slab statistics +// --------------- +// CAVEAT: This section describes statistics which are subject to change in the +// future. // -//The "stats" command with the argument of "slabs" returns information about -//each of the slabs created by memcached during runtime. This includes per-slab -//information along with some totals. The data is returned in the format: +// The "stats" command with the argument of "slabs" returns information about +// each of the slabs created by memcached during runtime. This includes per-slab +// information along with some totals. The data is returned in the format: // -//STAT : \r\n -//STAT \r\n +// STAT : \r\n +// STAT \r\n // -//The server terminates this list with the line +// The server terminates this list with the line: // -//END\r\n -func (c *Client) statsSlabsFromAddr(sss *StatsSlabsServer, mux *sync.Mutex, addr net.Addr) (err error) { - var wg sync.WaitGroup +// END\r\n +func (c *Client) statsSlabsFromAddr(sss *StatsSlabsServer, addr net.Addr) (err error) { err = c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { if _, err := fmt.Fprintf(rw, "stats slabs\r\n"); err != nil { return err @@ -154,91 +150,44 @@ func (c *Client) statsSlabsFromAddr(sss *StatsSlabsServer, mux *sync.Mutex, addr if bytes.Equal(line, resultEnd) { return err } - //STAT : \r\n - //STAT \r\n - wg.Add(1) - go func() { - sss.parseStat(string(line[5:len(line)-2]), mux) - wg.Done() - }() + // STAT : \r\n + // STAT \r\n + sss.parseStat(line[5 : len(line)-2]) } }) - wg.Wait() sss.ServerErr = err return } -func (sss *StatsSlabsServer) parseStat(line string, mux *sync.Mutex) { - // - tkns := strings.Split(string(line), ":") - if len(tkns) == 1 { - tkns := strings.Split(string(line), " ") - mux.Lock() - if tkns[0] == "active_slabs" { - sss.ActiveSlabs, _ = strconv.ParseUint(tkns[1], 10, 64) +func (sss *StatsSlabsServer) parseStat(line []byte) { + // : + // + tkns := bytes.FieldsFunc(line, func(r rune) bool { + return r == ':' || r == ' ' + }) + if len(tkns) == 2 { + if string(tkns[0]) == "active_slabs" { + sss.ActiveSlabs, _ = strconv.ParseUint(string(tkns[1]), 10, 64) } else { - sss.TotalMalloced, _ = strconv.ParseUint(tkns[1], 10, 64) + sss.TotalMalloced, _ = strconv.ParseUint(string(tkns[1]), 10, 64) } - mux.Unlock() return } - //: - - statValue := strings.Split(tkns[1], " ") - mux.Lock() - defer mux.Unlock() - slab := sss.StatsSlabs[tkns[0]] + slabclass := string(tkns[0]) + // + slab := sss.StatsSlabs[slabclass] if slab == nil { slab = new(StatsSlab) - sss.StatsSlabs[tkns[0]] = slab + slab.UnknownStats = make(map[string]string) + sss.StatsSlabs[slabclass] = slab } - slab.setStatValue(statValue[0], statValue[1]) -} - -func (slab *StatsSlab) setStatValue(stat, value string) { - v, err := getUint64Val(value) + err := parseSetStatValue(reflect.ValueOf(slab), string(tkns[1]), string(tkns[2])) if err != nil { - err = errors.New("Unable to parse value for stat" + stat + " stat " + err.Error()) - slab.Errs.AppendError(err) - return - } - switch stat { - case "chunk_size": - slab.ChunkSize = v - case "chunks_per_page": - slab.ChunksPerPage = v - case "total_pages": - slab.TotalPages = v - case "total_chunks": - slab.TotalChunks = v - case "get_hits": - slab.GetHits = v - case "cmd_set": - slab.CmdSet = v - case "delete_hits": - slab.DeleteHits = v - case "incr_hits": - slab.IncrHits = v - case "decr_hits": - slab.DecrHits = v - case "cas_hits": - slab.CasHits = v - case "cas_badval": - slab.CasBadval = v - case "touch_hits": - slab.TouchHits = v - case "used_chunks": - slab.UsedChunks = v - case "free_chunks": - slab.FreeChunks = v - case "free_chunks_end": - slab.FreeChunksEnd = v - case "mem_requested": - slab.MemRequested = v - case "active_slabs": - slab.ActiveSlabs = v - case "total_malloced": - slab.TotalMalloced = v + if err != errUnknownStat { + slab.Errs.AppendError(err) + } else { + slab.UnknownStats[string(tkns[1])] = string(tkns[2]) + } } }