From ca490c442be985bd0de0af06a19e8950e14f72b6 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Wed, 10 Feb 2016 17:26:30 +0000 Subject: [PATCH 001/631] core.memory: Rewrite memory.c in Lua using ljsyscall Switch to allocating HugeTLB pages as mmaped files on hugetlbfs instead of POSIX shared memory. The advantage of POSIX shared memory was that we did not need to rely on having a hugetlbfs filesystem mounted -- but now we auto-mount one at /var/run/snabb/hugetlbfs. This design is a bit simpler and should also make it easier for one Snabb process to map a page that was allocated by a different process. The main motivation for rewriting memory.c is to make it easier to maintain. Lua code is more in keeping with the rest of Snabb and it doesn't require people to be expert C programmers and debate the merits of POSIX compliance and using GCC extensions and so on. (It also puts a bit more distance between Snabb and monstrosities like autoconf which cannot be a bad thing.) --- src/core/memory.c | 106 -------------------------------------------- src/core/memory.h | 4 -- src/core/memory.lua | 81 +++++++++++++++++++++++++++++++-- 3 files changed, 77 insertions(+), 114 deletions(-) delete mode 100644 src/core/memory.c delete mode 100644 src/core/memory.h diff --git a/src/core/memory.c b/src/core/memory.c deleted file mode 100644 index e6d95350ba..0000000000 --- a/src/core/memory.c +++ /dev/null @@ -1,106 +0,0 @@ -// memory.c -- allocate dma-friendly memory -// -// Allocate HugeTLB memory pages for DMA. HugeTLB memory is always -// mapped to a virtual address with a specific scheme: -// -// virtual_address = physical_address | 0x500000000000ULL -// -// This makes it possible to resolve physical addresses directly from -// virtual addresses (remove the tag bits) and to test addresses for -// validity (check the tag bits). - -#define _GNU_SOURCE 1 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "memory.h" - -// Convert from virtual addresses in our own process address space to -// physical addresses in the RAM chips. -uint64_t virtual_to_physical(void *ptr) -{ - uint64_t virt_page; - static int pagemap_fd; - virt_page = ((uint64_t)ptr) / 4096; - if (pagemap_fd == 0) { - if ((pagemap_fd = open("/proc/self/pagemap", O_RDONLY)) <= 0) { - perror("open pagemap"); - return 0; - } - } - uint64_t data; - int len; - len = pread(pagemap_fd, &data, sizeof(data), virt_page * sizeof(uint64_t)); - if (len != sizeof(data)) { - perror("pread"); - return 0; - } - if ((data & (1ULL<<63)) == 0) { - fprintf(stderr, "page %lx not present: %lx", virt_page, data); - return 0; - } - return (data & ((1ULL << 55) - 1)) * 4096; -} - -// Map a new HugeTLB page to an appropriate virtual address. -// -// The HugeTLB page is allocated and mapped using the shm (shared -// memory) API. This API makes it easy to remap the page to a new -// virtual address after we resolve the physical address. -// -// There are two other APIs for allocating HugeTLB pages but they do -// not work as well: -// -// mmap() anonymous page with MAP_HUGETLB: cannot remap the address -// after allocation because Linux mremap() does not seem to work on -// HugeTLB pages. -// -// mmap() with file-backed MAP_HUGETLB: the file has to be on a -// hugetlbfs mounted filesystem and that is not necessarily -// available. -// -// Happily the shm API is happy to remap a HugeTLB page with an -// additional call to shmat() and does not depend on hugetlbfs. -// -// Further reading: -// https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt -// http://stackoverflow.com/questions/27997934/mremap2-with-hugetlb-to-change-virtual-address -void *allocate_huge_page(int size) -{ - int shmid = -1; - uint64_t physical_address, virtual_address; - void *tmpptr = MAP_FAILED; // initial kernel assigned virtual address - void *realptr = MAP_FAILED; // remapped virtual address - shmid = shmget(IPC_PRIVATE, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); - tmpptr = shmat(shmid, NULL, 0); - if (tmpptr == MAP_FAILED) { goto fail; } - if (mlock(tmpptr, size) != 0) { goto fail; } - physical_address = virtual_to_physical(tmpptr); - if (physical_address == 0) { goto fail; } - virtual_address = physical_address | 0x500000000000ULL; - realptr = shmat(shmid, (void*)virtual_address, 0); - if (realptr == MAP_FAILED) { goto fail; } - if (mlock(realptr, size) != 0) { goto fail; } - memset(realptr, 0, size); // zero memory to avoid potential surprises - shmdt(tmpptr); - shmctl(shmid, IPC_RMID, 0); - return realptr; - fail: - if (tmpptr != MAP_FAILED) { shmdt(tmpptr); } - if (realptr != MAP_FAILED) { shmdt(realptr); } - if (shmid != -1) { shmctl(shmid, IPC_RMID, 0); } - return NULL; -} - diff --git a/src/core/memory.h b/src/core/memory.h deleted file mode 100644 index 635a973c4b..0000000000 --- a/src/core/memory.h +++ /dev/null @@ -1,4 +0,0 @@ -int lock_memory(); -void *allocate_huge_page(int size); -uint64_t phys_page(uint64_t virt_page); - diff --git a/src/core/memory.lua b/src/core/memory.lua index cf080974c3..e5f99e49f5 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -11,7 +11,6 @@ local C = ffi.C local syscall = require("syscall") local lib = require("core.lib") -require("core.memory_h") --- ### Serve small allocations from hugepage "chunks" @@ -49,7 +48,7 @@ end function allocate_hugetlb_chunk () for i =1, 3 do - local page = C.allocate_huge_page(huge_page_size) + local page = allocate_huge_page(huge_page_size) if page ~= nil then return page else reserve_new_page() end end end @@ -85,8 +84,23 @@ huge_page_size = get_huge_page_size() -- Address bits per huge page (2MB = 21 bits; 1GB = 30 bits) huge_page_bits = math.log(huge_page_size, 2) ---- ### Physical address translation +--- Physical memory allocation +-- Allocate HugeTLB memory pages for DMA. HugeTLB memory is always +-- mapped to a virtual address with a specific scheme: +-- +-- virtual_address = physical_address | 0x500000000000ULL +-- +-- This makes it possible to resolve physical addresses directly from +-- virtual addresses (remove the tag bits) and to test addresses for +-- validity (check the tag bits). + +-- Tag applied to physical addresses to calculate virtual address. +local tag = 0x500000000000ULL + +-- virtual_to_physical(ptr) => uint64_t +-- +-- Return the physical address of specially mapped DMA memory. local uint64_t = ffi.typeof("uint64_t") function virtual_to_physical (virt_addr) local u64 = ffi.cast(uint64_t, virt_addr) @@ -97,13 +111,72 @@ function virtual_to_physical (virt_addr) return bit.bxor(u64, 0x500000000000ULL) end +-- function allocate_huge_page(size[, persistent]): +-- +-- Map a new HugeTLB page to an appropriate virtual address. +-- +-- The page is allocated via the hugetlbfs filesystem +-- /var/run/snabb/hugetlbfs that is mounted automatically. +-- The page has to be file-backed because the Linux kernel seems to +-- not support remap() on anonymous pages. +-- +-- Further reading: +-- https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt +-- http://stackoverflow.com/questions/27997934/mremap2-with-hugetlb-to-change-virtual-address +function allocate_huge_page (size, persistent) + ensure_hugetlbfs() + local tmpfile = "/var/run/snabb/hugetlbfs/alloc."..syscall.getpid() + local fd = syscall.open(tmpfile, "creat, rdwr", "RWXU") + assert(fd, "create hugetlb") + assert(syscall.ftruncate(fd, size), "ftruncate") + local tmpptr = syscall.mmap(nil, size, "read, write", "shared, hugetlb", fd, 0) + assert(tmpptr, "mmap hugetlb") + assert(syscall.mlock(tmpptr, size)) + local phys = resolve_physical(tmpptr) + local virt = bit.bor(phys, tag) + local ptr = syscall.mmap(virt, size, "read, write", "shared, hugetlb, fixed", fd, 0) + if persistent then + assert(syscall.rename(tmpfile, "/var/run/snabb/hugetlbfs/dma.%012x", phys)) + else + assert(syscall.unlink(tmpfile)) + end + syscall.close(fd) + return ptr +end + +-- resolve_physical(ptr) => uint64_t +-- +-- Resolve the physical address of the given pointer via the kernel. +function resolve_physical (ptr) + local pagesize = 4096 + local virtpage = ffi.cast("uint64_t", ptr) / pagesize + local pagemapfd = assert(syscall.open("/proc/self/pagemap", "rdonly")) + local data = ffi.new("uint64_t[1]") + syscall.pread(pagemapfd, data, 8, virtpage * 8) + syscall.close(pagemapfd) + assert(bit.band(data[0], bit.lshift(1, 63)) ~= 0ULL, "page not present") + local physpage = bit.band(data[0], 0xFFFFFFFFFFFFF) + return physpage * pagesize +end + +-- Make sure that /var/run/snabb/hugetlbfs is mounted. +function ensure_hugetlbfs () + syscall.mkdir("/var/run/snabb/hugetlbfs") + if not syscall.mount("none", "/var/run/snabb/hugetlbfs", "hugetlbfs", "rw,nosuid,nodev,noexec,relatime,remount") then + io.write("[mounting /var/run/snabb/hugetlbfs]\n") + assert(syscall.mount("none", "/var/run/snabb/hugetlbfs", "hugetlbfs", "rw,nosuid,nodev,noexec,relatime"), + "failed to (re)mount /var/run/snabb/hugetlbfs") + end +end + --- ### selftest function selftest (options) print("selftest: memory") print("Kernel vm.nr_hugepages: " .. syscall.sysctl("vm.nr_hugepages")) + ensure_hugetlbfs() -- can print a message, let that go first for i = 1, 4 do - io.write(" Allocating a "..(huge_page_size/1024/1024).."MB HugeTLB: ") + io.write(" Allocating a "..(huge_page_size/1024/1024).."MB HugeTLB:") io.flush() local dmaptr, physptr, dmalen = dma_alloc(huge_page_size) print("Got "..(dmalen/1024^2).."MB") From 05f94d77ebca1c1810688b240d1fbe89c00ffbba Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 18 Apr 2016 15:56:24 +0200 Subject: [PATCH 002/631] Move binary_search.dasl to core In anticipation of adding support for streaming lookups to CTable, move binary search implementation to core, and update users. --- src/apps/lwaftr/podhashmap.lua | 2 +- src/apps/lwaftr/rangemap.lua | 2 +- src/{apps/lwaftr => core}/binary_search.dasl | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/{apps/lwaftr => core}/binary_search.dasl (100%) diff --git a/src/apps/lwaftr/podhashmap.lua b/src/apps/lwaftr/podhashmap.lua index a8daa79ce8..c1fea27076 100644 --- a/src/apps/lwaftr/podhashmap.lua +++ b/src/apps/lwaftr/podhashmap.lua @@ -351,7 +351,7 @@ function PodHashMap:make_lookup_streamer(stride) -- These requires are here because they rely on dynasm, which the -- user might not have. In that case, since they get no benefit from -- streaming lookup, restrict them to the scalar lookup. - local binary_search = require('apps.lwaftr.binary_search') + local binary_search = require('core.binary_search') local multi_copy = require('apps.lwaftr.multi_copy') local res = { diff --git a/src/apps/lwaftr/rangemap.lua b/src/apps/lwaftr/rangemap.lua index f77d76efa0..0d2aa88276 100644 --- a/src/apps/lwaftr/rangemap.lua +++ b/src/apps/lwaftr/rangemap.lua @@ -12,7 +12,7 @@ module(..., package.seeall) local ffi = require("ffi") local C = ffi.C -local binary_search = require('apps.lwaftr.binary_search') +local binary_search = require('core.binary_search') local UINT32_MAX = 0xFFFFFFFF diff --git a/src/apps/lwaftr/binary_search.dasl b/src/core/binary_search.dasl similarity index 100% rename from src/apps/lwaftr/binary_search.dasl rename to src/core/binary_search.dasl From 21d4e74de4e1c2a51c3683dfbd739d99cbb48590 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 18 Apr 2016 16:04:57 +0200 Subject: [PATCH 003/631] Move multi_copy.dasl to core In preparation for supporting streaming lookup in CTable, move the specialized parallel copy routine to core. --- src/apps/lwaftr/podhashmap.lua | 2 +- src/{apps/lwaftr => core}/multi_copy.dasl | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{apps/lwaftr => core}/multi_copy.dasl (100%) diff --git a/src/apps/lwaftr/podhashmap.lua b/src/apps/lwaftr/podhashmap.lua index a8daa79ce8..7812bfb15d 100644 --- a/src/apps/lwaftr/podhashmap.lua +++ b/src/apps/lwaftr/podhashmap.lua @@ -352,7 +352,7 @@ function PodHashMap:make_lookup_streamer(stride) -- user might not have. In that case, since they get no benefit from -- streaming lookup, restrict them to the scalar lookup. local binary_search = require('apps.lwaftr.binary_search') - local multi_copy = require('apps.lwaftr.multi_copy') + local multi_copy = require('core.multi_copy') local res = { all_entries = self.entries, diff --git a/src/apps/lwaftr/multi_copy.dasl b/src/core/multi_copy.dasl similarity index 100% rename from src/apps/lwaftr/multi_copy.dasl rename to src/core/multi_copy.dasl From 6608e9e4bd68c0270edbd43ec11a832a1e3df948 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 18 Apr 2016 16:11:38 +0200 Subject: [PATCH 004/631] Whoops, binary_search to lib/, not core/ It was all a misunderstanding, you see. --- src/apps/lwaftr/podhashmap.lua | 2 +- src/apps/lwaftr/rangemap.lua | 2 +- src/{core => lib}/binary_search.dasl | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/{core => lib}/binary_search.dasl (100%) diff --git a/src/apps/lwaftr/podhashmap.lua b/src/apps/lwaftr/podhashmap.lua index c1fea27076..ba1db275b5 100644 --- a/src/apps/lwaftr/podhashmap.lua +++ b/src/apps/lwaftr/podhashmap.lua @@ -351,7 +351,7 @@ function PodHashMap:make_lookup_streamer(stride) -- These requires are here because they rely on dynasm, which the -- user might not have. In that case, since they get no benefit from -- streaming lookup, restrict them to the scalar lookup. - local binary_search = require('core.binary_search') + local binary_search = require('lib.binary_search') local multi_copy = require('apps.lwaftr.multi_copy') local res = { diff --git a/src/apps/lwaftr/rangemap.lua b/src/apps/lwaftr/rangemap.lua index 0d2aa88276..78e364556a 100644 --- a/src/apps/lwaftr/rangemap.lua +++ b/src/apps/lwaftr/rangemap.lua @@ -12,7 +12,7 @@ module(..., package.seeall) local ffi = require("ffi") local C = ffi.C -local binary_search = require('core.binary_search') +local binary_search = require('lib.binary_search') local UINT32_MAX = 0xFFFFFFFF diff --git a/src/core/binary_search.dasl b/src/lib/binary_search.dasl similarity index 100% rename from src/core/binary_search.dasl rename to src/lib/binary_search.dasl From 6cff17c0d7e222427cf238397df7403de752b870 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 18 Apr 2016 16:13:47 +0200 Subject: [PATCH 005/631] Whoops, multi_copy to lib/, not core/ --- src/apps/lwaftr/podhashmap.lua | 2 +- src/{core => lib}/multi_copy.dasl | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{core => lib}/multi_copy.dasl (100%) diff --git a/src/apps/lwaftr/podhashmap.lua b/src/apps/lwaftr/podhashmap.lua index 7812bfb15d..b129786c95 100644 --- a/src/apps/lwaftr/podhashmap.lua +++ b/src/apps/lwaftr/podhashmap.lua @@ -352,7 +352,7 @@ function PodHashMap:make_lookup_streamer(stride) -- user might not have. In that case, since they get no benefit from -- streaming lookup, restrict them to the scalar lookup. local binary_search = require('apps.lwaftr.binary_search') - local multi_copy = require('core.multi_copy') + local multi_copy = require('lib.multi_copy') local res = { all_entries = self.entries, diff --git a/src/core/multi_copy.dasl b/src/lib/multi_copy.dasl similarity index 100% rename from src/core/multi_copy.dasl rename to src/lib/multi_copy.dasl From b5cc8c101618611dd26863f71b77629a816d1514 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 18 Apr 2016 17:02:14 +0200 Subject: [PATCH 006/631] Add streaming lookup to CTable I documented the streaming lookup code, such as it is. Probably it needs to be better though! --- src/lib/README.ctable.md | 60 ++++++++++++++++++++ src/lib/ctable.lua | 120 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 179 insertions(+), 1 deletion(-) diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md index d0178212a5..f05ebaa774 100644 --- a/src/lib/README.ctable.md +++ b/src/lib/README.ctable.md @@ -174,6 +174,66 @@ for entry in ctab:iterate() do end ``` +#### Streaming interface + +As mentioned earlier, batching multiple lookups can amortize the cost +of a round-trip to RAM. To do this, first prepare a `LookupStreamer` +for the batch size that you need. You will have to experiment to find +the batch size that works best for your table's entry sizes; for +reference, for 32-byte entries a 32-wide lookup seems to be optimum. + +```lua +-- Stream in 32 lookups at once. +local stride = 32 +local streamer = ctab:make_lookup_streamer(stride) +``` + +Wiring up streaming lookup in a packet-processing network is a bit of +a chore currently, as you have to maintain separate queues of lookup +keys and packets, assuming that each lookup maps to a packet. Let's +make a little helper: + +```lua +local lookups = { + queue = ffi.new("struct packet * [?]", stride), + queue_len = 0, + streamer = streamer +} + +local function flush(lookups) + if lookups.queue_len > 0 then + -- Here is the magic! + lookups.streamer:stream() + for i = 0, lookups.queue_len - 1 do + local pkt = lookups.queue[i] + if lookups.streamer:is_found(i) + local val = lookups.streamer.entries[i].value + --- Do something cool here! + end + end + lookups.queue_len = 0 + end +end + +local function enqueue(lookups, pkt, key) + local n = lookups.queue_len + lookups.streamer.entries[n].key = key + lookups.queue[n] = pkt + n = n + 1 + if n == stride then + flush(lookups) + else + lookups.queue_len = n + end +end +``` + +Then as you see packets, you enqueue them via `enqueue`, extracting +out the key from the packet in some way and passing that value as the +argument. When `enqueue` detects that the queue is full, it will +flush it, performing the lookups in parallel and processing the +results. + #### Hash functions Any hash function will do, as long as it produces values in the `[0, diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 53e71c32db..ff803b96db 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -4,11 +4,14 @@ local ffi = require("ffi") local C = ffi.C local S = require("syscall") local bit = require("bit") +local binary_search = require("lib.binary_search") +local multi_copy = require("lib.multi_copy") local bxor, bnot = bit.bxor, bit.bnot local tobit, lshift, rshift = bit.tobit, bit.lshift, bit.rshift local max, floor, ceil = math.max, math.floor, math.ceil CTable = {} +LookupStreamer = {} local HASH_MAX = 0xFFFFFFFF local uint16_ptr_t = ffi.typeof('uint16_t*') @@ -292,6 +295,103 @@ function CTable:remove(key, missing_allowed) return true end +function CTable:make_lookup_streamer(stride) + local res = { + all_entries = self.entries, + stride = stride, + hash_fn = self.hash_fn, + equal_fn = self.equal_fn, + entries_per_lookup = self.max_displacement + 1, + scale = self.scale, + pointers = ffi.new('void*['..stride..']'), + entries = self.type(stride), + -- Binary search over N elements can return N if no entry was + -- found that was greater than or equal to the key. We would + -- have to check the result of binary search to ensure that we + -- are reading a value in bounds. To avoid this, allocate one + -- more entry. + stream_entries = self.type(stride * (self.max_displacement + 1) + 1) + } + -- Give res.pointers sensible default values in case the first lookup + -- doesn't fill the pointers vector. + for i = 0, stride-1 do res.pointers[i] = self.entries end + + -- Initialize the stream_entries to HASH_MAX for sanity. + for i = 0, stride * (self.max_displacement + 1) do + res.stream_entries[i].hash = HASH_MAX + end + + -- Compile multi-copy and binary-search procedures that are + -- specialized for this table and this stride. + local entry_size = ffi.sizeof(self.entry_type) + res.multi_copy = multi_copy.gen(stride, res.entries_per_lookup * entry_size) + res.binary_search = binary_search.gen(res.entries_per_lookup, self.entry_type) + + return setmetatable(res, { __index = LookupStreamer }) +end + +function LookupStreamer:stream() + local stride = self.stride + local entries = self.entries + local pointers = self.pointers + local stream_entries = self.stream_entries + local entries_per_lookup = self.entries_per_lookup + local equal_fn = self.equal_fn + + for i=0,stride-1 do + local hash = self.hash_fn(entries[i].key) + local index = hash_to_index(hash, self.scale) + entries[i].hash = hash + pointers[i] = self.all_entries + index + end + + self.multi_copy(stream_entries, pointers) + + -- Copy results into entries. + for i=0,stride-1 do + local hash = entries[i].hash + local index = i * entries_per_lookup + local found = self.binary_search(stream_entries + index, hash) + -- It could be that we read one beyond the ENTRIES_PER_LOOKUP + -- entries allocated for this key; that's fine. See note in + -- make_lookup_streamer. + if found.hash == hash then + -- Direct hit? + if equal_fn(found.key, entries[i].key) then + entries[i].value = found.value + else + -- Mark this result as not found unless we prove + -- otherwise. + entries[i].hash = HASH_MAX + + -- Collision? + found = found + 1 + while found.hash == hash do + if equal_fn(found.key, entries[i].key) then + -- Yay! Re-mark this result as found. + entries[i].hash = hash + entries[i].value = found.value + break + end + found = found + 1 + end + end + else + -- Not found. + entries[i].hash = HASH_MAX + end + end +end + +function LookupStreamer:is_empty(i) + assert(i >= 0 and i < self.stride) + return self.entries[i].hash == HASH_MAX +end + +function LookupStreamer:is_found(i) + return not self:is_empty(i) +end + function CTable:selfcheck() local occupancy = 0 local max_displacement = 0 @@ -444,7 +544,25 @@ function selftest() for entry in ctab:iterate() do iterated = iterated + 1 end assert(iterated == occupancy) - -- OK, all looking good with our ctab. + -- OK, all looking good with the normal interfaces; let's check out + -- streaming lookup. + local stride = 1 + repeat + local streamer = ctab:make_lookup_streamer(stride) + for i = 1, occupancy, stride do + local n = math.min(stride, occupancy-i+1) + for j = 0, n-1 do + streamer.entries[j].key = i + j + end + streamer:stream() + for j = 0, n-1 do + assert(streamer:is_found(j)) + local value = streamer.entries[j].value[0] + assert(value == bnot(i + j)) + end + end + stride = stride * 2 + until stride > 256 -- A check that our equality functions work as intended. local numbers_equal = make_equal_fn(ffi.typeof('int')) From 0f13d757903cd76de1fd69d2a116223aae2ff3dd Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 25 Apr 2016 11:09:03 +0200 Subject: [PATCH 007/631] Initial commit of YANG parser --- src/lib/yang/parser.lua | 253 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 src/lib/yang/parser.lua diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua new file mode 100644 index 0000000000..0cf32e1476 --- /dev/null +++ b/src/lib/yang/parser.lua @@ -0,0 +1,253 @@ +module(..., package.seeall) + +local lib = require('core.lib') + +local function new_type() + local Type = {} + Type.__index = Type + return Type +end + +local Leaf = new_type() +function Leaf.new(properties) + return setmetatable(properties, Leaf) +end + +local Parser = new_type() +function Parser.new(str, filename) + local ret = { pos=1, str=str, filename=filename, line=1, column=0, line_pos=1} + ret = setmetatable(ret, Parser) + ret.peek_char = ret:read_char() + return ret +end + +function Parser:error(msg, ...) + print(self.str:match("[^\n]*", self.line_pos)) + print(string.rep(" ", self.column).."^") + error(('%s:%d:%d: error: '..msg):format( + self.name or '', self.line, self.column, ...)) +end + +function Parser:read_char() + if self.pos <= #self.str then + local ret = self.str:sub(self.pos,self.pos) + self.pos = self.pos + 1 + return ret + end +end + +function Parser:peek() return self.peek_char end +function Parser:is_eof() return not self:peek() end + +function Parser:next() + local chr = self.peek_char + if chr == '\n' then + self.line_pos = self.pos + 1 + self.column = 0 + self.line = self.line + 1 + elseif char == "\t" then + self.column = self.column + 8 + self.column = 8 * math.floor(self.column / 8) + elseif chr then + self.column = self.column + 1 + end + self.peek_char = self:read_char() + return chr +end + +function Parser:check(expected) + if self:peek() == expected then + if expected then self:next() end + return true + end + return false +end + +function Parser:consume(expected) + if not self:check(expected) then + local ch = self:peek() + if ch == nil then + self:error("while looking for '%s', got EOF", expected) + elseif expected then + self:error("expected '%s', got '%s'", expected, ch) + else + self:error("expected EOF, got '%s'", ch) + end + end +end + +function Parser:take_while(pattern) + local res = {} + while not self:is_eof() and self:peek():match(pattern) do + table.insert(res, self:next()) + end + return table.concat(res) +end + +-- Returns true if has consumed any whitespace +function Parser:skip_whitespace() + local result = false + if self:take_while('%s') ~= "" then result = true end + -- Skip comments, which start with # and continue to the end of line. + while self:check('#') do + result = true + self:take_while('[^\n]') + self:take_while('%s') + end + return result +end + +function Parser:consume_whitespace() + if not self:skip_whitespace() then + self:error("Missing whitespace") + end +end + +function Parser:consume_token(pattern, expected) + local tok = self:take_while(pattern) + if tok:lower() ~= expected then + self:error("expected '%s', got '%s'", expected, tok) + end +end + +function Parser:parse_qstring(quote) + local start_column = self.column + self:check(quote) + local terminators = "\n"..quote + if quote == '"' then terminators = terminators.."\\" end + + local result = "" + while true do + result = result..self:take_while("[^"..terminators.."]") + if self:check(quote) then break end + if self:check("\n") then + while self.column < start_column do + if not self:check(" ") and not self:check("\t") then break end + end + result = result.."\n" + if self.column > start_column then + result = result..stirng.rep(" ", self.column-start_column) + end + elseif self:check("\\") then + if self:check("n") then result = result.."\n" + elseif self:check("t") then result = result.."\t" + elseif self:check('"') then result = result..'"' + elseif self:check("\\") then result = result.."\\" + else self:error("Invalid escaped character") end + end + end + self:check(quote) + self:skip_whitespace() + + if not self:check("+") then return result end + self:skip_whitespace() + + -- Strings can be concaternated together with a + + if self:check("'") then + return result..self:parse_qstring("'") + elseif self:check('"') then + return result..self:parse_qstring('"') + else + self:error("Expected quote character") + end +end + +function Parser:parse_identifier() + local id + if self:check("'") then id = self:parse_qstirng("'") + elseif self:check('"') then id = self:parse_qstring('"') + else id = self:take_while("[%w_.-]") end + + if not id == "" then self:error("Expected identifier") end + if not id:match("^[%a_]") then self:error("Invalid identifier") end + + return id +end + +function Parser:parse_keyword() + self:skip_whitespace() + + if self:is_eof() then + self:error("Expected keyword") + end + + local char = self:peek() + local is_prefix = char == "'" or char == '"' + local id = self:parse_identifier() + + if self:check(":") then + local extension_id = self:parse_identifier() + return {id, extension_id} + end + + if is_prefix then error("Expected colon") end + + return id +end + +function Parser:parse_statement() + self:consume_whitespace() + + -- Then must be a string that is the leaf's identifier + local leaf_identifier = self:take_while("%a") + if leaf_identifier == "" then + self:error("Leaf identifier expected") + end + self:skip_whitespace() + + -- Consume the opening curly brace. + self:consume("{") + self:skip_whitespace() + + -- Next we have the property name, some whitespace then the value. + local properties = {} + while not self:expect("}") do + -- Read in the property name + local property = self:take_while("%a+") + self:consume_whitespace() + + -- Take the value + local value = self:take_while("[%w:]") + + -- Check there is a new line (can they also be seperated by commas?) + local line = self.line + self:skip_whitespace() + if self.line == line then + self:error("properties should be split by a new line") + end + end + + return Leaf.new(properties) +end + +function selftest() + local function assert_equal(a, b) + if not lib.equal(a, b) then + print(a, b) + error("not equal") + end + end + + assert(getmetatable(Leaf.new({})) == Leaf) + + local parser = Parser.new("foo", "bar") + assert_equal(parser:next(), "f") + assert(parser:next() == "o") + assert(parser:next() == "o") + assert(parser:is_eof()) + assert(not parser:next()) + + -- Test tsyes' code + local parser = Parser.new([[ + leaf foo { + type string; + } + ]], "bar") + -- for now lets just consume the leaf keyword here. + parser:skip_whitespace() + parser:consume_token("%a", "leaf") + local leaf = parser:parse_leaf() + + assert(leaf.type == "string") + +end From 0e3495efd8b06e3ed46475b5c9d194248e43842f Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 25 Apr 2016 11:08:26 +0200 Subject: [PATCH 008/631] Add statement parser --- src/lib/yang/example.yang | 1188 +++++++++++++++++++++++++++++++++++++ src/lib/yang/parser.lua | 198 +++++-- 2 files changed, 1331 insertions(+), 55 deletions(-) create mode 100644 src/lib/yang/example.yang diff --git a/src/lib/yang/example.yang b/src/lib/yang/example.yang new file mode 100644 index 0000000000..aab5041ac6 --- /dev/null +++ b/src/lib/yang/example.yang @@ -0,0 +1,1188 @@ +/* + * This module has been generated by smidump 0.4.8: + * + * smidump -f yang IF-MIB + * + * Do not edit. Edit the source file instead! + */ + +module IF-MIB { + + /*** NAMESPACE / PREFIX DEFINITION ***/ + + namespace "urn:ietf:params:xml:ns:yang:smiv2:IF-MIB"; + prefix "if-mib"; + + /*** LINKAGE (IMPORTS / INCLUDES) ***/ + + import IANAifType-MIB { prefix "ianaiftype-mib"; } + import SNMPv2-TC { prefix "smiv2"; } + import ietf-yang-types { prefix "yang"; } + + /*** META INFORMATION ***/ + + organization + "IETF Interfaces MIB Working Group"; + + contact + " Keith McCloghrie + Cisco Systems, Inc. + 170 West Tasman Drive + San Jose, CA 95134-1706 + US + + 408-526-5260 + kzm@cisco.com"; + + description + "The MIB module to describe generic objects for network + interface sub-layers. This MIB is an updated version of + MIB-II's ifTable, and incorporates the extensions defined in + RFC 1229."; + + revision "2000-06-14" { + description + "Clarifications agreed upon by the Interfaces MIB WG, and + published as RFC 2863."; + } + revision "1996-02-28" { + description + "Revisions made by the Interfaces MIB WG, and published in + RFC 2233."; + } + revision "1993-11-08" { + description + "Initial revision, published as part of RFC 1573."; + } + + /*** TYPE DEFINITIONS ***/ + + typedef OwnerString { + type string { + length "0..255"; + pattern "\p{IsBasicLatin}{0,255}"; + } + status deprecated; + description + "This data type is used to model an administratively + assigned name of the owner of a resource. This information + is taken from the NVT ASCII character set. It is suggested + that this name contain one or more of the following: ASCII + form of the manager station's transport address, management + station name (e.g., domain name), network management + personnel's name, location, or phone number. In some cases + the agent itself will be the owner of an entry. In these + cases, this string shall be set to a string starting with + 'agent'."; + } + + typedef InterfaceIndex { + type int32 { + range "1..2147483647"; + } + description + "A unique value, greater than zero, for each interface or + interface sub-layer in the managed system. It is + recommended that values are assigned contiguously starting + from 1. The value for each interface sub-layer must remain + constant at least from one re-initialization of the entity's + network management system to the next re-initialization."; + } + + typedef InterfaceIndexOrZero { + type int32 { + range "0..2147483647"; + } + description + "This textual convention is an extension of the + InterfaceIndex convention. The latter defines a greater + than zero value used to identify an interface or interface + sub-layer in the managed system. This extension permits the + additional value of zero. the value zero is object-specific + and must therefore be defined as part of the description of + any object which uses this syntax. Examples of the usage of + zero might include situations where interface was unknown, + or when none or all interfaces need to be referenced."; + } + + container interfaces { + + leaf ifNumber { + type int32; + config false; + description + "The number of network interfaces (regardless of their + current state) present on this system."; + } + + + /* XXX table comments here XXX */ + + list ifEntry { + key "ifIndex"; + description + "An entry containing management information applicable to a + particular interface."; + + + leaf ifIndex { + type if-mib:InterfaceIndex; + description + "A unique value, greater than zero, for each interface. It + is recommended that values are assigned contiguously + starting from 1. The value for each interface sub-layer + must remain constant at least from one re-initialization of + the entity's network management system to the next re- + initialization."; + } + + leaf ifDescr { + type smiv2:DisplayString { + length "0..255"; + } + config false; + description + "A textual string containing information about the + interface. This string should include the name of the + manufacturer, the product name and the version of the + interface hardware/software."; + } + + leaf ifType { + type ianaiftype-mib:IANAifType; + config false; + description + "The type of interface. Additional values for ifType are + assigned by the Internet Assigned Numbers Authority (IANA), + through updating the syntax of the IANAifType textual + convention."; + } + + leaf ifMtu { + type int32; + config false; + description + "The size of the largest packet which can be sent/received + on the interface, specified in octets. For interfaces that + are used for transmitting network datagrams, this is the + size of the largest network datagram that can be sent on the + interface."; + } + + leaf ifSpeed { + type yang:gauge32; + config false; + description + "An estimate of the interface's current bandwidth in bits + per second. For interfaces which do not vary in bandwidth + or for those where no accurate estimation can be made, this + object should contain the nominal bandwidth. If the + bandwidth of the interface is greater than the maximum value + reportable by this object then this object should report its + maximum value (4,294,967,295) and ifHighSpeed must be used + to report the interace's speed. For a sub-layer which has + no concept of bandwidth, this object should be zero."; + } + + leaf ifPhysAddress { + type yang:phys-address; + config false; + description + "The interface's address at its protocol sub-layer. For + example, for an 802.x interface, this object normally + contains a MAC address. The interface's media-specific MIB + must define the bit and byte ordering and the format of the + value of this object. For interfaces which do not have such + an address (e.g., a serial line), this object should contain + an octet string of zero length."; + } + + leaf ifAdminStatus { + type enumeration { + enum up { value 1; } + enum down { value 2; } + enum testing { value 3; } + } + config true; + description + "The desired state of the interface. The testing(3) state + indicates that no operational packets can be passed. When a + managed system initializes, all interfaces start with + ifAdminStatus in the down(2) state. As a result of either + explicit management action or per configuration information + retained by the managed system, ifAdminStatus is then + changed to either the up(1) or testing(3) states (or remains + in the down(2) state)."; + } + + leaf ifOperStatus { + type enumeration { + enum up { value 1; } + enum down { value 2; } + enum testing { value 3; } + enum unknown { value 4; } + enum dormant { value 5; } + enum notPresent { value 6; } + enum lowerLayerDown { value 7; } + } + config false; + description + "The current operational state of the interface. The + testing(3) state indicates that no operational packets can + be passed. If ifAdminStatus is down(2) then ifOperStatus + should be down(2). If ifAdminStatus is changed to up(1) + then ifOperStatus should change to up(1) if the interface is + ready to transmit and receive network traffic; it should + change to dormant(5) if the interface is waiting for + external actions (such as a serial line waiting for an + incoming connection); it should remain in the down(2) state + if and only if there is a fault that prevents it from going + to the up(1) state; it should remain in the notPresent(6) + state if the interface has missing (typically, hardware) + components."; + } + + leaf ifLastChange { + type yang:timeticks; + config false; + description + "The value of sysUpTime at the time the interface entered + its current operational state. If the current state was + entered prior to the last re-initialization of the local + network management subsystem, then this object contains a + zero value."; + } + + leaf ifInOctets { + type yang:counter32; + config false; + description + "The total number of octets received on the interface, + + + including framing characters. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifInUcastPkts { + type yang:counter32; + config false; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, which were not addressed to a multicast + or broadcast address at this sub-layer. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifInNUcastPkts { + type yang:counter32; + config false; + status deprecated; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, which were addressed to a multicast or + broadcast address at this sub-layer. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime. + + This object is deprecated in favour of ifInMulticastPkts and + ifInBroadcastPkts."; + } + + leaf ifInDiscards { + type yang:counter32; + config false; + description + "The number of inbound packets which were chosen to be + discarded even though no errors had been detected to prevent + + + their being deliverable to a higher-layer protocol. One + possible reason for discarding such a packet could be to + free up buffer space. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifInErrors { + type yang:counter32; + config false; + description + "For packet-oriented interfaces, the number of inbound + packets that contained errors preventing them from being + deliverable to a higher-layer protocol. For character- + oriented or fixed-length interfaces, the number of inbound + transmission units that contained errors preventing them + from being deliverable to a higher-layer protocol. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifInUnknownProtos { + type yang:counter32; + config false; + description + "For packet-oriented interfaces, the number of packets + received via the interface which were discarded because of + an unknown or unsupported protocol. For character-oriented + or fixed-length interfaces that support protocol + multiplexing the number of transmission units received via + the interface which were discarded because of an unknown or + unsupported protocol. For any interface that does not + support protocol multiplexing, this counter will always be + 0. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifOutOctets { + type yang:counter32; + config false; + description + "The total number of octets transmitted out of the + interface, including framing characters. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifOutUcastPkts { + type yang:counter32; + config false; + description + "The total number of packets that higher-level protocols + requested be transmitted, and which were not addressed to a + multicast or broadcast address at this sub-layer, including + those that were discarded or not sent. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifOutNUcastPkts { + type yang:counter32; + config false; + status deprecated; + description + "The total number of packets that higher-level protocols + requested be transmitted, and which were addressed to a + multicast or broadcast address at this sub-layer, including + those that were discarded or not sent. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime. + + This object is deprecated in favour of ifOutMulticastPkts + and ifOutBroadcastPkts."; + } + + leaf ifOutDiscards { + type yang:counter32; + config false; + description + "The number of outbound packets which were chosen to be + discarded even though no errors had been detected to prevent + their being transmitted. One possible reason for discarding + such a packet could be to free up buffer space. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifOutErrors { + type yang:counter32; + config false; + description + "For packet-oriented interfaces, the number of outbound + packets that could not be transmitted because of errors. + For character-oriented or fixed-length interfaces, the + number of outbound transmission units that could not be + transmitted because of errors. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifOutQLen { + type yang:gauge32; + config false; + status deprecated; + description + "The length of the output packet queue (in packets)."; + } + + leaf ifSpecific { + type yang:object-identifier; + config false; + status deprecated; + description + "A reference to MIB definitions specific to the particular + media being used to realize the interface. It is + + + recommended that this value point to an instance of a MIB + object in the media-specific MIB, i.e., that this object + have the semantics associated with the InstancePointer + textual convention defined in RFC 2579. In fact, it is + recommended that the media-specific MIB specify what value + ifSpecific should/can take for values of ifType. If no MIB + definitions specific to the particular media are available, + the value should be set to the OBJECT IDENTIFIER { 0 0 }."; + } + } + } + + container ifMIBObjects { + + + /* XXX table comments here XXX */ + + list ifStackEntry { + key "ifStackHigherLayer ifStackLowerLayer"; + description + "Information on a particular relationship between two sub- + layers, specifying that one sub-layer runs on 'top' of the + other sub-layer. Each sub-layer corresponds to a conceptual + row in the ifTable."; + + + leaf ifStackHigherLayer { + type if-mib:InterfaceIndexOrZero; + description + "The value of ifIndex corresponding to the higher sub-layer + of the relationship, i.e., the sub-layer which runs on 'top' + of the sub-layer identified by the corresponding instance of + ifStackLowerLayer. If there is no higher sub-layer (below + the internetwork layer), then this object has the value 0."; + } + + leaf ifStackLowerLayer { + type if-mib:InterfaceIndexOrZero; + description + "The value of ifIndex corresponding to the lower sub-layer + of the relationship, i.e., the sub-layer which runs 'below' + the sub-layer identified by the corresponding instance of + ifStackHigherLayer. If there is no lower sub-layer, then + this object has the value 0."; + } + + leaf ifStackStatus { + type smiv2:RowStatus; + config true; + description + "The status of the relationship between two sub-layers. + + Changing the value of this object from 'active' to + 'notInService' or 'destroy' will likely have consequences up + and down the interface stack. Thus, write access to this + object is likely to be inappropriate for some types of + interfaces, and many implementations will choose not to + support write-access for any type of interface."; + } + } + + + + /* XXX table comments here XXX */ + + list ifRcvAddressEntry { + key "ifIndex ifRcvAddressAddress"; + description + "A list of objects identifying an address for which the + system will accept packets/frames on the particular + interface identified by the index value ifIndex."; + + leaf ifIndex { + type leafref { + path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; + } + description + "Automagically generated leafref leaf."; + } + + leaf ifRcvAddressAddress { + type yang:phys-address; + description + "An address for which the system will accept packets/frames + on this entry's interface."; + } + + leaf ifRcvAddressStatus { + type smiv2:RowStatus; + config true; + description + "This object is used to create and delete rows in the + ifRcvAddressTable."; + } + + leaf ifRcvAddressType { + type enumeration { + enum other { value 1; } + enum volatile { value 2; } + enum nonVolatile { value 3; } + } + config true; + description + "This object has the value nonVolatile(3) for those entries + in the table which are valid and will not be deleted by the + next restart of the managed system. Entries having the + value volatile(2) are valid and exist, but have not been + saved, so that will not exist after the next restart of the + managed system. Entries having the value other(1) are valid + and exist but are not classified as to whether they will + continue to exist after the next restart."; + } + } + + leaf ifTableLastChange { + type yang:timeticks; + config false; + description + "The value of sysUpTime at the time of the last creation or + deletion of an entry in the ifTable. If the number of + entries has been unchanged since the last re-initialization + of the local network management subsystem, then this object + contains a zero value."; + } + + leaf ifStackLastChange { + type yang:timeticks; + config false; + description + "The value of sysUpTime at the time of the last change of + the (whole) interface stack. A change of the interface + stack is defined to be any creation, deletion, or change in + value of any instance of ifStackStatus. If the interface + stack has been unchanged since the last re-initialization of + the local network management subsystem, then this object + contains a zero value."; + } + } + + + /* XXX table comments here XXX */ + + augment "/if-mib:interfaces/if-mib:ifEntry" { + description + "An entry containing additional management information + applicable to a particular interface."; + + leaf ifName { + type smiv2:DisplayString; + config false; + description + "The textual name of the interface. The value of this + object should be the name of the interface as assigned by + the local device and should be suitable for use in commands + entered at the device's `console'. This might be a text + name, such as `le0' or a simple port number, such as `1', + depending on the interface naming syntax of the device. If + several entries in the ifTable together represent a single + interface as named by the device, then each will have the + same value of ifName. Note that for an agent which responds + to SNMP queries concerning an interface on some other + (proxied) device, then the value of ifName for such an + interface is the proxied device's local name for it. + + If there is no local name, or this object is otherwise not + applicable, then this object contains a zero-length string."; + } + + leaf ifInMulticastPkts { + type yang:counter32; + config false; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, which were addressed to a multicast + address at this sub-layer. For a MAC layer protocol, this + includes both Group and Functional addresses. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + + + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifInBroadcastPkts { + type yang:counter32; + config false; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, which were addressed to a broadcast + address at this sub-layer. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifOutMulticastPkts { + type yang:counter32; + config false; + description + "The total number of packets that higher-level protocols + requested be transmitted, and which were addressed to a + multicast address at this sub-layer, including those that + were discarded or not sent. For a MAC layer protocol, this + includes both Group and Functional addresses. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifOutBroadcastPkts { + type yang:counter32; + config false; + description + "The total number of packets that higher-level protocols + requested be transmitted, and which were addressed to a + broadcast address at this sub-layer, including those that + were discarded or not sent. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + + + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCInOctets { + type yang:counter64; + config false; + description + "The total number of octets received on the interface, + including framing characters. This object is a 64-bit + version of ifInOctets. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCInUcastPkts { + type yang:counter64; + config false; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, which were not addressed to a multicast + or broadcast address at this sub-layer. This object is a + 64-bit version of ifInUcastPkts. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCInMulticastPkts { + type yang:counter64; + config false; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, which were addressed to a multicast + address at this sub-layer. For a MAC layer protocol, this + includes both Group and Functional addresses. This object + is a 64-bit version of ifInMulticastPkts. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCInBroadcastPkts { + type yang:counter64; + config false; + description + "The number of packets, delivered by this sub-layer to a + higher (sub-)layer, which were addressed to a broadcast + address at this sub-layer. This object is a 64-bit version + of ifInBroadcastPkts. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCOutOctets { + type yang:counter64; + config false; + description + "The total number of octets transmitted out of the + interface, including framing characters. This object is a + 64-bit version of ifOutOctets. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCOutUcastPkts { + type yang:counter64; + config false; + description + "The total number of packets that higher-level protocols + requested be transmitted, and which were not addressed to a + multicast or broadcast address at this sub-layer, including + those that were discarded or not sent. This object is a + 64-bit version of ifOutUcastPkts. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCOutMulticastPkts { + type yang:counter64; + config false; + description + "The total number of packets that higher-level protocols + requested be transmitted, and which were addressed to a + multicast address at this sub-layer, including those that + were discarded or not sent. For a MAC layer protocol, this + includes both Group and Functional addresses. This object + is a 64-bit version of ifOutMulticastPkts. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifHCOutBroadcastPkts { + type yang:counter64; + config false; + description + "The total number of packets that higher-level protocols + requested be transmitted, and which were addressed to a + broadcast address at this sub-layer, including those that + were discarded or not sent. This object is a 64-bit version + of ifOutBroadcastPkts. + + Discontinuities in the value of this counter can occur at + re-initialization of the management system, and at other + times as indicated by the value of + ifCounterDiscontinuityTime."; + } + + leaf ifLinkUpDownTrapEnable { + type enumeration { + enum enabled { value 1; } + enum disabled { value 2; } + } + config true; + description + "Indicates whether linkUp/linkDown traps should be generated + for this interface. + + By default, this object should have the value enabled(1) for + interfaces which do not operate on 'top' of any other + interface (as defined in the ifStackTable), and disabled(2) + otherwise."; + } + + leaf ifHighSpeed { + type yang:gauge32; + config false; + description + "An estimate of the interface's current bandwidth in units + of 1,000,000 bits per second. If this object reports a + value of `n' then the speed of the interface is somewhere in + the range of `n-500,000' to `n+499,999'. For interfaces + which do not vary in bandwidth or for those where no + accurate estimation can be made, this object should contain + the nominal bandwidth. For a sub-layer which has no concept + of bandwidth, this object should be zero."; + } + + leaf ifPromiscuousMode { + type smiv2:TruthValue; + config true; + description + "This object has a value of false(2) if this interface only + accepts packets/frames that are addressed to this station. + This object has a value of true(1) when the station accepts + all packets/frames transmitted on the media. The value + true(1) is only legal on certain types of media. If legal, + setting this object to a value of true(1) may require the + interface to be reset before becoming effective. + + The value of ifPromiscuousMode does not affect the reception + of broadcast and multicast packets/frames by the interface."; + } + + leaf ifConnectorPresent { + type smiv2:TruthValue; + config false; + description + "This object has the value 'true(1)' if the interface + sublayer has a physical connector and the value 'false(2)' + otherwise."; + } + + leaf ifAlias { + type smiv2:DisplayString { + length "0..64"; + } + config true; + description + "This object is an 'alias' name for the interface as + specified by a network manager, and provides a non-volatile + 'handle' for the interface. + + On the first instantiation of an interface, the value of + ifAlias associated with that interface is the zero-length + string. As and when a value is written into an instance of + ifAlias through a network management set operation, then the + agent must retain the supplied value in the ifAlias instance + associated with the same interface for as long as that + interface remains instantiated, including across all re- + initializations/reboots of the network management system, + including those which result in a change of the interface's + ifIndex value. + + An example of the value which a network manager might store + in this object for a WAN interface is the (Telco's) circuit + number/identifier of the interface. + + Some agents may support write-access only for interfaces + having particular values of ifType. An agent which supports + write access to this object is required to keep the value in + non-volatile storage, but it may limit the length of new + values depending on how much storage is already occupied by + the current values for other interfaces."; + } + + leaf ifCounterDiscontinuityTime { + type yang:timestamp; + config false; + description + "The value of sysUpTime on the most recent occasion at which + any one or more of this interface's counters suffered a + discontinuity. The relevant counters are the specific + instances associated with this interface of any Counter32 or + + + Counter64 object contained in the ifTable or ifXTable. If + no such discontinuities have occurred since the last re- + initialization of the local management subsystem, then this + object contains a zero value."; + } + } + + + /* XXX table comments here XXX */ + + augment "/if-mib:interfaces/if-mib:ifEntry" { + status deprecated; + description + "An entry containing objects for invoking tests on an + interface."; + + leaf ifTestId { + type smiv2:TestAndIncr; + config true; + status deprecated; + description + "This object identifies the current invocation of the + interface's test."; + } + + leaf ifTestStatus { + type enumeration { + enum notInUse { value 1; } + enum inUse { value 2; } + } + config true; + status deprecated; + description + "This object indicates whether or not some manager currently + has the necessary 'ownership' required to invoke a test on + this interface. A write to this object is only successful + when it changes its value from 'notInUse(1)' to 'inUse(2)'. + After completion of a test, the agent resets the value back + to 'notInUse(1)'."; + } + + leaf ifTestType { + type smiv2:AutonomousType; + config true; + status deprecated; + description + "A control variable used to start and stop operator- + initiated interface tests. Most OBJECT IDENTIFIER values + assigned to tests are defined elsewhere, in association with + specific types of interface. However, this document assigns + a value for a full-duplex loopback test, and defines the + special meanings of the subject identifier: + + noTest OBJECT IDENTIFIER ::= { 0 0 } + + When the value noTest is written to this object, no action + is taken unless a test is in progress, in which case the + test is aborted. Writing any other value to this object is + + + only valid when no test is currently in progress, in which + case the indicated test is initiated. + + When read, this object always returns the most recent value + that ifTestType was set to. If it has not been set since + the last initialization of the network management subsystem + on the agent, a value of noTest is returned."; + } + + leaf ifTestResult { + type enumeration { + enum none { value 1; } + enum success { value 2; } + enum inProgress { value 3; } + enum notSupported { value 4; } + enum unAbleToRun { value 5; } + enum aborted { value 6; } + enum failed { value 7; } + } + config false; + status deprecated; + description + "This object contains the result of the most recently + requested test, or the value none(1) if no tests have been + requested since the last reset. Note that this facility + provides no provision for saving the results of one test + when starting another, as could be required if used by + multiple managers concurrently."; + } + + leaf ifTestCode { + type yang:object-identifier; + config false; + status deprecated; + description + "This object contains a code which contains more specific + information on the test result, for example an error-code + after a failed test. Error codes and other values this + object may take are specific to the type of interface and/or + test. The value may have the semantics of either the + AutonomousType or InstancePointer textual conventions as + defined in RFC 2579. The identifier: + + testCodeUnknown OBJECT IDENTIFIER ::= { 0 0 } + + is defined for use if no additional result code is + available."; + } + + leaf ifTestOwner { + type if-mib:OwnerString; + config true; + status deprecated; + description + "The entity which currently has the 'ownership' required to + invoke a test on this interface."; + } + } + + notification linkDown { + description + "A linkDown trap signifies that the SNMP entity, acting in + an agent role, has detected that the ifOperStatus object for + one of its communication links is about to enter the down + state from some other state (but not from the notPresent + state). This other state is indicated by the included value + of ifOperStatus."; + + container linkDown-ifIndex { + leaf ifIndex { + type leafref { + path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; + } + description + "Automagically generated leafref leaf."; + } + } + + container linkDown-ifAdminStatus { + leaf ifIndex { + type leafref { + path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; + } + description + "Automagically generated leafref leaf."; + } + leaf ifAdminStatus { + type enumeration { + enum up { value 1; } + enum down { value 2; } + enum testing { value 3; } + } + description + "The desired state of the interface. The testing(3) state + indicates that no operational packets can be passed. When a + managed system initializes, all interfaces start with + ifAdminStatus in the down(2) state. As a result of either + explicit management action or per configuration information + retained by the managed system, ifAdminStatus is then + changed to either the up(1) or testing(3) states (or remains + in the down(2) state)."; + } + } + + container linkDown-ifOperStatus { + leaf ifIndex { + type leafref { + path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; + } + description + "Automagically generated leafref leaf."; + } + leaf ifOperStatus { + type enumeration { + enum up { value 1; } + enum down { value 2; } + enum testing { value 3; } + enum unknown { value 4; } + enum dormant { value 5; } + enum notPresent { value 6; } + enum lowerLayerDown { value 7; } + } + description + "The current operational state of the interface. The + testing(3) state indicates that no operational packets can + be passed. If ifAdminStatus is down(2) then ifOperStatus + should be down(2). If ifAdminStatus is changed to up(1) + then ifOperStatus should change to up(1) if the interface is + ready to transmit and receive network traffic; it should + change to dormant(5) if the interface is waiting for + external actions (such as a serial line waiting for an + incoming connection); it should remain in the down(2) state + if and only if there is a fault that prevents it from going + to the up(1) state; it should remain in the notPresent(6) + state if the interface has missing (typically, hardware) + components."; + } + } + + } + + notification linkUp { + description + "A linkUp trap signifies that the SNMP entity, acting in an + agent role, has detected that the ifOperStatus object for + one of its communication links left the down state and + transitioned into some other state (but not into the + notPresent state). This other state is indicated by the + included value of ifOperStatus."; + + container linkUp-ifIndex { + leaf ifIndex { + type leafref { + path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; + } + description + "Automagically generated leafref leaf."; + } + } + + container linkUp-ifAdminStatus { + leaf ifIndex { + type leafref { + path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; + } + description + "Automagically generated leafref leaf."; + } + leaf ifAdminStatus { + type enumeration { + enum up { value 1; } + enum down { value 2; } + enum testing { value 3; } + } + description + "The desired state of the interface. The testing(3) state + indicates that no operational packets can be passed. When a + managed system initializes, all interfaces start with + ifAdminStatus in the down(2) state. As a result of either + explicit management action or per configuration information + retained by the managed system, ifAdminStatus is then + changed to either the up(1) or testing(3) states (or remains + in the down(2) state)."; + } + } + + container linkUp-ifOperStatus { + leaf ifIndex { + type leafref { + path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; + } + description + "Automagically generated leafref leaf."; + } + leaf ifOperStatus { + type enumeration { + enum up { value 1; } + enum down { value 2; } + enum testing { value 3; } + enum unknown { value 4; } + enum dormant { value 5; } + enum notPresent { value 6; } + enum lowerLayerDown { value 7; } + } + description + "The current operational state of the interface. The + testing(3) state indicates that no operational packets can + be passed. If ifAdminStatus is down(2) then ifOperStatus + should be down(2). If ifAdminStatus is changed to up(1) + then ifOperStatus should change to up(1) if the interface is + ready to transmit and receive network traffic; it should + change to dormant(5) if the interface is waiting for + external actions (such as a serial line waiting for an + incoming connection); it should remain in the down(2) state + if and only if there is a fault that prevents it from going + to the up(1) state; it should remain in the notPresent(6) + state if the interface has missing (typically, hardware) + components."; + } + } + + } +} /* end of module IF-MIB */ \ No newline at end of file diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 0cf32e1476..4977f31ce7 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -45,7 +45,7 @@ function Parser:next() self.line_pos = self.pos + 1 self.column = 0 self.line = self.line + 1 - elseif char == "\t" then + elseif chr == "\t" then self.column = self.column + 8 self.column = 8 * math.floor(self.column / 8) elseif chr then @@ -84,15 +84,26 @@ function Parser:take_while(pattern) return table.concat(res) end +function Parser:skip_c_comment() + repeat + self:take_while("[^*]") + self:consume("*") + until self:check("/") +end + -- Returns true if has consumed any whitespace function Parser:skip_whitespace() local result = false if self:take_while('%s') ~= "" then result = true end -- Skip comments, which start with # and continue to the end of line. - while self:check('#') do + while self:check('/') do result = true - self:take_while('[^\n]') - self:take_while('%s') + if self:check("*") then self:skip_c_comment() + else + self:consume("/") + self:take_while('[^\n]') + self:take_while('%s') + end end return result end @@ -126,14 +137,17 @@ function Parser:parse_qstring(quote) end result = result.."\n" if self.column > start_column then - result = result..stirng.rep(" ", self.column-start_column) + result = result..string.rep(" ", self.column-start_column) end elseif self:check("\\") then if self:check("n") then result = result.."\n" elseif self:check("t") then result = result.."\t" elseif self:check('"') then result = result..'"' elseif self:check("\\") then result = result.."\\" - else self:error("Invalid escaped character") end + else + result = result.."\\" + end + end end self:check(quote) @@ -152,15 +166,16 @@ function Parser:parse_qstring(quote) end end -function Parser:parse_identifier() - local id - if self:check("'") then id = self:parse_qstirng("'") - elseif self:check('"') then id = self:parse_qstring('"') - else id = self:take_while("[%w_.-]") end +function Parser:parse_string() + if self:check("'") then return self:parse_qstring("'") + elseif self:check('"') then return self:parse_qstring('"') + else return self:take_while("[^%s;{}\"'/]") end +end +function Parser:parse_identifier() + local id = self:parse_string() if not id == "" then self:error("Expected identifier") end - if not id:match("^[%a_]") then self:error("Invalid identifier") end - + if not id:match("^[%a_][%w_.-]*$") then self:error("Invalid identifier") end return id end @@ -181,43 +196,60 @@ function Parser:parse_keyword() end if is_prefix then error("Expected colon") end - return id end +function Parser:parse_module() + local statements = self:parse_statement_list() + if not self:is_eof() then error("Not end of file") end + return statements +end + +function Parser:parse_statement_list() + local statements = {} + + while true do + self:skip_whitespace() + if self:is_eof() or self:peek() == "}" then + break + end + + table.insert(statements, self:parse_statement()) + end + + return statements + +end + function Parser:parse_statement() - self:consume_whitespace() + self:skip_whitespace() + + local returnval = {} -- Then must be a string that is the leaf's identifier - local leaf_identifier = self:take_while("%a") - if leaf_identifier == "" then - self:error("Leaf identifier expected") + local keyword = self:parse_keyword() + if keyword == "" then + self:error("keyword expected") end - self:skip_whitespace() + returnval.keyword = keyword + self:consume_whitespace() - -- Consume the opening curly brace. - self:consume("{") + -- Take the identifier + local argument = self:parse_string() + if argument ~= "" then returnval.argument = argument end self:skip_whitespace() - -- Next we have the property name, some whitespace then the value. - local properties = {} - while not self:expect("}") do - -- Read in the property name - local property = self:take_while("%a+") - self:consume_whitespace() - - -- Take the value - local value = self:take_while("[%w:]") + if self:check(";") then + return returnval + end - -- Check there is a new line (can they also be seperated by commas?) - local line = self.line - self:skip_whitespace() - if self.line == line then - self:error("properties should be split by a new line") - end + if self:check("{") then + returnval.statements = self:parse_statement_list() + self:consume("}") + return returnval end - return Leaf.new(properties) + self:error("Unexpected character found") end function selftest() @@ -228,26 +260,82 @@ function selftest() end end - assert(getmetatable(Leaf.new({})) == Leaf) + local function test_string(src, exp) + local parser = Parser.new(src) + parser:skip_whitespace() - local parser = Parser.new("foo", "bar") - assert_equal(parser:next(), "f") - assert(parser:next() == "o") - assert(parser:next() == "o") - assert(parser:is_eof()) - assert(not parser:next()) + assert_equal(parser:parse_string(), exp) + end + + local function pp(x) + if type(x) == "table" then + io.write("{") + local first = true + for k,v in pairs(x) do + if not first then + io.write(", ") + end + io.write(k.."=") + pp(v) + first = false + end + io.write("}") + elseif type(x) == "string" then + io.write(x) + else + error("Unsupported type") + end + end - -- Test tsyes' code - local parser = Parser.new([[ - leaf foo { - type string; - } - ]], "bar") - -- for now lets just consume the leaf keyword here. - parser:skip_whitespace() - parser:consume_token("%a", "leaf") - local leaf = parser:parse_leaf() - assert(leaf.type == "string") + local function test_module(src, exp) + local parser = Parser.new(src) + local result = parser:parse_module() + if not lib.equal(result, exp) then + pp(result) + pp(exp) + error("no equal") + end + end + + local function lines(...) + return table.concat({...}, "\n") + end + -- Test the string parser + test_string("foo", "foo") + test_string([["foo"]], "foo") + test_string([["foo"+"bar"]], "foobar") + test_string([['foo'+"bar"]], "foobar") + test_string("'foo\\nbar'", "foo\\nbar") + test_string('"foo\\nbar"', "foo\nbar") + test_string('"// foo bar;"', '// foo bar;') + test_string('"/* foo bar */"', '/* foo bar */') + test_string([["foo \"bar\""]], 'foo "bar"') + test_string(lines(" 'foo", + " bar'"), + lines("foo", " bar")) + test_string(lines(" 'foo", + " bar'"), + lines("foo", "bar")) + test_string(lines(" 'foo", + "\tbar'"), + lines("foo", " bar")) + test_string(lines(" 'foo", + " bar'"), + lines("foo", "bar")) + + + test_module("type string;", {{keyword="type", argument="string"}}) + test_module("/** **/", {}) + test_module("// foo bar;", {}) + test_module("// foo bar;\nleaf port;", {{keyword="leaf", argument="port"}}) + test_module("type/** hellooo */string;", {{keyword="type", argument="string"}}) + test_module('type "hello\\pq";', {{keyword="type", argument="hello\\pq"}}) + + + local fin = assert(io.open("example.yang")) + local yangexample = fin:read("*a") + local parser = Parser.new(yangexample, "example.yang") + parser:parse_module() end From 447f2c5b0f0c8a529ddacaa69af958298ca816c1 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 25 Apr 2016 13:51:01 +0200 Subject: [PATCH 009/631] Add more testing, docs, and fix bug - Add more testing in selftest (cover all expected use cases) - Add documentation at the top explaining what one should expect - Fix a bug where a statement without an argument would raise an error --- src/lib/yang/parser.lua | 72 +++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 4977f31ce7..ffcbe74dce 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -1,22 +1,39 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +-- This module implements a YANG parser which will parse a YANG module to a +-- Lua table. This is used by instantiating the Parser with the module and +-- optionally the filename. You then should call Parser:parse_module +-- +-- The parser uses the same termanology as the specification (rfc6020). The +-- following is an example YANG module and lua representation: +-- YANG: +-- +-- leaf port { +-- type inet:port-number; +-- default; +-- description "The port to which the SSH server listens" +-- } +-- +-- Lua table: +-- { +-- {argument="port", keyword="leaf", statements={ +-- {keyword="type", argument="inet:port-number"}, +-- {keyword="default"}, +-- { +-- keyword="description", +-- argument="The port to which the SSH server listens" +-- }} +-- } +-- } + module(..., package.seeall) local lib = require('core.lib') -local function new_type() - local Type = {} - Type.__index = Type - return Type -end - -local Leaf = new_type() -function Leaf.new(properties) - return setmetatable(properties, Leaf) -end - -local Parser = new_type() +local Parser = {} function Parser.new(str, filename) - local ret = { pos=1, str=str, filename=filename, line=1, column=0, line_pos=1} - ret = setmetatable(ret, Parser) + local ret = {pos=1, str=str, filename=filename, line=1, column=0, line_pos=1} + ret = setmetatable(ret, {__index = Parser}) ret.peek_char = ret:read_char() return ret end @@ -226,15 +243,19 @@ function Parser:parse_statement() local returnval = {} - -- Then must be a string that is the leaf's identifier + -- Then must be a string that is the statement's identifier local keyword = self:parse_keyword() if keyword == "" then self:error("keyword expected") end returnval.keyword = keyword - self:consume_whitespace() + if self:check(";") then + -- We've ended the statement without an argument. + return returnval + end -- Take the identifier + self:consume_whitespace() local argument = self:parse_string() if argument ~= "" then returnval.argument = argument end self:skip_whitespace() @@ -312,26 +333,29 @@ function selftest() test_string('"// foo bar;"', '// foo bar;') test_string('"/* foo bar */"', '/* foo bar */') test_string([["foo \"bar\""]], 'foo "bar"') - test_string(lines(" 'foo", - " bar'"), + test_string(lines(" 'foo", " bar'"), lines("foo", " bar")) - test_string(lines(" 'foo", - " bar'"), + test_string(lines(" 'foo", " bar'"), lines("foo", "bar")) - test_string(lines(" 'foo", - "\tbar'"), + test_string(lines(" 'foo", "\tbar'"), lines("foo", " bar")) - test_string(lines(" 'foo", - " bar'"), + test_string(lines(" 'foo", " bar'"), lines("foo", "bar")) + test_module("type;", {{keyword="type"}}) test_module("type string;", {{keyword="type", argument="string"}}) test_module("/** **/", {}) test_module("// foo bar;", {}) test_module("// foo bar;\nleaf port;", {{keyword="leaf", argument="port"}}) test_module("type/** hellooo */string;", {{keyword="type", argument="string"}}) test_module('type "hello\\pq";', {{keyword="type", argument="hello\\pq"}}) + test_module(lines("leaf port {", "type number;", "}"), + {{keyword="leaf", argument="port", + statements={{keyword="type", argument="number"}}}}) + test_module(lines("leaf port {", "type;", "}"), + {{keyword="leaf", argument="port", + statements={{keyword="type"}}}}) local fin = assert(io.open("example.yang")) From a738eb215f31722c4b6f8c3461f81c0ca8ed06e3 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 25 Apr 2016 15:13:27 +0200 Subject: [PATCH 010/631] Fix syntax problem and add helper function --- src/lib/yang/parser.lua | 46 +++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index ffcbe74dce..8ea8e71ceb 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -273,6 +273,17 @@ function Parser:parse_statement() self:error("Unexpected character found") end +function parse_string(str, filename) + local parser = Parser.new(str, filename) + return parser:parse_module() +end + +function parse_file(filename) + local file_in = assert(io.open(filename)) + local contents = file_in:read("*a") + return parse_string(contents, filename) +end + function selftest() local function assert_equal(a, b) if not lib.equal(a, b) then @@ -310,8 +321,7 @@ function selftest() local function test_module(src, exp) - local parser = Parser.new(src) - local result = parser:parse_module() + local result = parse_string(src) if not lib.equal(result, exp) then pp(result) pp(exp) @@ -333,14 +343,10 @@ function selftest() test_string('"// foo bar;"', '// foo bar;') test_string('"/* foo bar */"', '/* foo bar */') test_string([["foo \"bar\""]], 'foo "bar"') - test_string(lines(" 'foo", " bar'"), - lines("foo", " bar")) - test_string(lines(" 'foo", " bar'"), - lines("foo", "bar")) - test_string(lines(" 'foo", "\tbar'"), - lines("foo", " bar")) - test_string(lines(" 'foo", " bar'"), - lines("foo", "bar")) + test_string(lines(" 'foo", " bar'"), lines("foo", " bar")) + test_string(lines(" 'foo", " bar'"), lines("foo", "bar")) + test_string(lines(" 'foo", "\tbar'"), lines("foo", " bar")) + test_string(lines(" 'foo", " bar'"), lines("foo", "bar")) test_module("type;", {{keyword="type"}}) @@ -350,16 +356,12 @@ function selftest() test_module("// foo bar;\nleaf port;", {{keyword="leaf", argument="port"}}) test_module("type/** hellooo */string;", {{keyword="type", argument="string"}}) test_module('type "hello\\pq";', {{keyword="type", argument="hello\\pq"}}) - test_module(lines("leaf port {", "type number;", "}"), - {{keyword="leaf", argument="port", - statements={{keyword="type", argument="number"}}}}) - test_module(lines("leaf port {", "type;", "}"), - {{keyword="leaf", argument="port", - statements={{keyword="type"}}}}) - - - local fin = assert(io.open("example.yang")) - local yangexample = fin:read("*a") - local parser = Parser.new(yangexample, "example.yang") - parser:parse_module() + test_module(lines("leaf port {", "type number;", "}"), {{keyword="leaf", + argument="port", statements={{keyword="type", argument="number"}}}}) + test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", + argument="port", statements={{keyword="type"}}}}) + + + -- Expects tests to be run from the "src" directory at the root of the repo + parse_file("lib/yang/example.yang") end From d5ced20d0e979241a58e65a7ccb9824ba0b1630a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 25 Apr 2016 15:28:21 +0200 Subject: [PATCH 011/631] Remove unnecessary newlines --- src/lib/yang/parser.lua | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 8ea8e71ceb..1f44cff976 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -224,18 +224,14 @@ end function Parser:parse_statement_list() local statements = {} - while true do self:skip_whitespace() if self:is_eof() or self:peek() == "}" then break end - table.insert(statements, self:parse_statement()) end - return statements - end function Parser:parse_statement() @@ -295,7 +291,6 @@ function selftest() local function test_string(src, exp) local parser = Parser.new(src) parser:skip_whitespace() - assert_equal(parser:parse_string(), exp) end @@ -347,8 +342,6 @@ function selftest() test_string(lines(" 'foo", " bar'"), lines("foo", "bar")) test_string(lines(" 'foo", "\tbar'"), lines("foo", " bar")) test_string(lines(" 'foo", " bar'"), lines("foo", "bar")) - - test_module("type;", {{keyword="type"}}) test_module("type string;", {{keyword="type", argument="string"}}) test_module("/** **/", {}) @@ -361,7 +354,6 @@ function selftest() test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type"}}}}) - -- Expects tests to be run from the "src" directory at the root of the repo parse_file("lib/yang/example.yang") end From 16a9713e524bb1be3b2b34a5bf9976b6acc03d3c Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 3 May 2016 11:54:47 +0200 Subject: [PATCH 012/631] Create a schema parser for YANG --- src/lib/yang/schema.lua | 514 ++++++++++++++++++++++++++++++++++++ src/lib/yang/validation.lua | 34 +++ 2 files changed, 548 insertions(+) create mode 100644 src/lib/yang/schema.lua create mode 100644 src/lib/yang/validation.lua diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua new file mode 100644 index 0000000000..8716bb30fe --- /dev/null +++ b/src/lib/yang/schema.lua @@ -0,0 +1,514 @@ +-- Built in types (page 19) +-- +-- +---------------------+-------------------------------------+ +-- | Name | Description | +-- +---------------------+-------------------------------------+ +-- | binary | Any binary data | +-- | bits | A set of bits or flags | +-- | boolean | "true" or "false" | +-- | decimal64 | 64-bit signed decimal number | +-- | empty | A leaf that does not have any value | +-- | enumeration | Enumerated strings | +-- | identityref | A reference to an abstract identity | +-- | instance-identifier | References a data tree node | +-- | int8 | 8-bit signed integer | +-- | int16 | 16-bit signed integer | +-- | int32 | 32-bit signed integer | +-- | int64 | 64-bit signed integer | +-- | leafref | A reference to a leaf instance | +-- | string | Human-readable string | +-- | uint8 | 8-bit unsigned integer | +-- | uint16 | 16-bit unsigned integer | +-- | uint32 | 32-bit unsigned integer | +-- | uint64 | 64-bit unsigned integer | +-- | union | Choice of member types | +-- +---------------------+-------------------------------------+ +-- +-- Module: +-- module { ... } +-- +-- required: namespace, prefix +-- optional: anyxml, augment, choice, contact, container, description, deviation, +-- extension, feature, grouping, identity, import, include, leaf, leaf-list +-- list, notification, organization, reference, revision, rpc, typedef, uses, +-- yang-version +-- +module(..., package.seeall) + +local validation = require("lib.yang.validation") + +-- Use ffi types because they will validate that numeric values are being +-- provided. The downside is that integer overflow could occur on these. This +-- route has been selected as validation will be faster than attempting to +-- validate in Lua. +local ffi = require("ffi") +ffi.cdef[[ +typedef struct { int8_t value; } int8box; +typedef struct { int16_t value; } int16box; +typedef struct { int32_t value; } int32box; +typedef struct { int64_t value; } int64box; +typedef struct { uint8_t value; } uint8box; +typedef struct { uint16_t value; } uint16box; +typedef struct { uint32_t value; } uint32box; +typedef struct { uint64_t value; } uint64box; +typedef struct { double value; } decimal64box; +]] + + +-- This is a boxed value for datatypes which are represeted in Lua and validated +-- in Lua. This includes everything except the numeric values. +local Box = {} +function Box.new(validate) + local ret = {realroot={}} + local mt = { + __newindex = function (t, k, v) + if validate then validate(v) end + ret.realroot[k] = v + end, + __index = function(t, k) + local v = ret.realroot[k] + return v + end, + } + return setmetatable(ret, mt) +end +-- Yang feature +local Feature = {} +function Feature.new(name) + local ret = {config={name=name}} + return setmetatable(ret, {__index = Feature}) +end + +function Feature:validate(statements) + local cardinality = {description={0,1}, status={0,1}, refernece={0,1}} + validation.cardinality("feature", name, cardinality, statements) +end + +function Feature:consume(statements) + if statements.description then + self.config.description = statements.description[1].argument + end + if statements.reference then + self.config.reference = statements.reference[1].argument + end + if statements.status then + self.config.status = statements.reference[1].argument + end +end + +-- Yang Leaf +local Leaf = {} +function Leaf.new(name) + local ret = {config={name=name}} + ret.config.iffeature = {} + return setmetatable(ret, {__index = Leaf}) +end + +function Leaf:validate(statements) + local cardinality = {config={0,1}, default={0,1}, description={0,1}, + mandatory={0,1}, reference={0,1}, status={0,1}, + type={1,1}, units={0,1}, when={0,1}} + validation.cardinality("leaf", name, cardinality, src) +end + +function Leaf:consume(statements) + if statements.default then self.config.default = statements.default end + if statements.description then + self.config.description = statements.description[1].argument + end + + if statements["if-feature"] then + for _, f in pairs(statements["if-feature"]) do + table.insert(self.config.iffconfig["if-feature"], f.argument) + end + end + + -- Handle the type which will involve picking the correct box + -- to store the value with validation if needed. + self.config.type = statements.type[1].argument + + -- First deal with built in types. + if self.config.type == "int8" then + self.box = ffi.new("int8box") + elseif self.config.type == "int16" then + self.box = ffi.new("int16box") + elseif self.config.type == "int32" then + self.box = ffi.new("int32box") + elseif self.config.type == "int64" then + self.box = ffi.new("int64box") + elseif self.config.type == "uint8" then + self.box = ffi.new("uint8box") + elseif self.config.type == "uint16" then + self.box = ffi.new("uint16box") + elseif self.config.type == "uint32" then + self.box = ffi.new("uint32box") + elseif self.config.type == "uint64" then + self.box = ffi.new("uint64box") + elseif self.config.type == "decimal64" then + self.box = ffi.new("decimal64box") + elseif self.config.type == "string" then + self.box = Box.new() + elseif self.config.type == "boolean" then + self.box = Box.new(function (value) + if type(value) ~= "boolean" then + error( + ("Value for '%s' must be true or false."):format(self.config.name)) + end + end) + else + error(("Unknown type '%s' for leaf '%s'"):format( + leaf_type, self.config.name)) + end +end + +-- Yang list +local List = {} +function List.new(name) + local ret = {config={name=name}} + return setmetatable(ret, {__index = List}) +end + +function List:validate(statements) + local cardinality = {config={0,1}, description={0,1}, key={0,1}, + reference={0,1}, status={0,1}, when={0,1}} + cardinality["max-elements"] = {0,1} + cardinality["min-elements"] = {0,1} + cardinality["ordered-by"] = {0,1} + validation.cardinality("list", name, cardinality, statements) +end + +function List:consume(statements) + if statements.key then + self.config.key = statements.key[1].argument + end + if statements.leaf then + for _, leaf in pairs(statements.leaf) do + self[leaf.argument] = Leaf.new(leaf.argument) + self[leaf.argument]:consume(leaf.statements) + end + end +end + +-- Yang group +local Grouping = {} +function Grouping.new(name) + local ret = {config={name=name}} + return setmetatable(ret, {__index = Grouping}) +end + +function Grouping:validate(statements) + local cardinality = {description={0,1}, reference={0,1}, status={0,1}} + validation.cardinality("grouping", name, cardinality, statements) +end + +function Grouping:consume(statements) + if statements.description then + self.config.description = statements.description[1].argument + end + + if statements.list then + for _, list in pairs(statements.list) do + self[list.argument] = List.new(list.argument) + self[list.argument]:consume(list.statements) + end + end + + if statements.leaf then + for _, leaf in pairs(statements.leaf) do + self[leaf.argument] = Leaf.new(leaf.argument) + self[leaf.argument]:consume(leaf.statements) + end + end +end + +-- Yang Revision +local Revision = {} +function Revision.new(date) + local ret = {config={date=date}} + return setmetatable(ret, {__index = Revision}) +end + +function Revision:validate(statements) + local cardinality = {description={0,1}, reference={0,1}} + validation.cardinality("revision", self.config.date, cardinality, statements) +end + +function Revision:consume(statements) + self:validate(statements) + + if statements.description then + self.config.description = statements.description[1].argument + end + if statements.description then + self.config.reference = statements.reference.argument + end +end + +-- Yang Module +local Module = {} +function Module.new(name) + local ret = {body={}, config={name=name}, modules={}, revisions={}, + features={}, groupings={}} + return setmetatable(ret, {__index = Module}) +end + +function Module:load() + -- Find the file and load it. + print("DEBUG: loading module", self.config.name) +end + +function Module:consume(statements) + -- Validate the statements first. + self:validate(statements) + + -- Set the meta information about the module + self.config.namespace = statements.namespace[1].argument + self.config.prefix = statements.prefix[1].argument + + if statements.organization then + self.config.organization = statements.organization[1].argument + end + if statements.contact then + self.config.contact = statements.contact[1].argument + end + + if statements.description then + self.config.description = statements.description[1].argument + end + + -- Now handle the imports, as other things may reference them. + if statements.import then + for _, mod in pairs(statements.import) do + self.modules[mod.argument] = Module.new(mod.argument) + + -- Ask the module to find and load itself. + self.modules[mod.argument]:load() + end + end + + -- Handle revisions + if statements.revision then + for _, revision in pairs(statements.revision) do + -- TODO: can or should we convert these revision date stamps. + self.revisions[revision.argument] = Revision.new(revision.argument) + self.revisions[revision.argument]:consume(revision.statements) + end + end + + -- Feature statements + if statements.feature then + for _, feature in pairs(statements.feature) do + self.features[feature.argument] = Feature.new(feature.argument) + self.features[feature.argument]:consume(feature.statements) + end + end + + -- Leaf statements + if statements.leaf then + for _, leaf in pairs(statements.revision) do + end + end + + -- List statements + if statements.grouping then + for _, grouping in pairs(statements.grouping) do + self.groupings[grouping.argument] = Grouping.new(grouping.argument) + self.groupings[grouping.argument]:consume(grouping.statements) + end + end +end + +function Module:validate(statements) + local cardinality = {contact={0,1}, description={0,1}, namespace={1,1}, + organization={0,1}, prefix={1,1}, reference={0,1}} + cardinality["yang-version"] = {0,1} + validation.cardinality("module", self.name, cardinality, statements) +end + + +function extract_nodes(structure) + local nodes = {} + for _, v in pairs(structure) do + -- Recursively apply this. + if v.statements then v.statements = extract_nodes(v.statements) end + + -- Add to the nodes table. + if nodes[v.keyword] then + table.insert(nodes[v.keyword], v) + else + nodes[v.keyword] = {v} + end + end + return nodes +end + +function load_schema(src) + -- Okay conver the schema into something more useful. + src = extract_nodes(src) + + -- Okay we're expecting a module at the top + if not src.module then error("Expected 'module'") end + local mod = Module.new(src.module[1].argument) + mod:consume(src.module[1].statements) + return mod +end + + +function selftest() + local test_schema = [[module ietf-softwire { + namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; + prefix "softwire"; + + import ietf-inet-types {prefix inet; } + import ietf-yang-types {prefix yang; } + + organization "Softwire Working Group"; + + contact + " + Qi Sun sunqi.ietf@gmail.com + Hao Wang wangh13@mails.tsinghua.edu.cn + Yong Cui yong@csnet1.cs.tsinghua.edu.cn + Ian Farrer ian.farrer@telekom.de + Mohamed Boucadair mohamed.boucadair@orange.com + Rajiv Asati rajiva@cisco.com + "; + + description + "This document defines a YANG data model for the configuration and + management of IPv4-in-IPv6 Softwire Border Routers and Customer + Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T + Softwire mechanisms. + + Copyright (c) 2014 IETF Trust and the persons identified + as authors of the code. All rights reserved. + This version of this YANG module is part of RFC XXX; see the RFC + itself for full legal notices."; + + + revision 2015-09-30 { + description + "Version-04: fix YANG syntax; Add flags to map-rule; Remove + the map-rule-type element. "; + reference "tbc"; + } + + revision 2015-04-07 { + description + "Version-03: Integrate lw4over6; Updata state nodes; Correct + grammar errors; Reuse groupings; Update descriptions. + Simplify the model."; + reference "tbc"; + } + + revision 2015-02-10 { + description + "Version-02: Add notifications."; + reference "tbc"; + } + + + revision 2015-02-06 { + description + "Version-01: Correct grammar errors; Reuse groupings; Update + descriptions."; + reference "tbc"; + } + + revision 2015-02-02 { + description + "Initial revision."; + reference "tbc"; + } + + feature lw4over6 { + description + "Lightweight 4over6 moves the Network Address and Port + Translation (NAPT) function from the centralized DS-Lite tunnel + concentrator to the tunnel client located in the Customer + Premises Equipment (CPE). This removes the requirement for a + Carrier Grade NAT function in the tunnel concentrator and + reduces the amount of centralized state that must be held to a + per-subscriber level. In order to delegate the NAPT function + and make IPv4 Address sharing possible, port-restricted IPv4 + addresses are allocated to the CPEs."; + reference + "I-D.ietf-softwire-lw4over6"; + } + + feature map-e { + description + "MAP-E is a mechanism for transporting IPv4 packets across an + IPv6 network using IP encapsulation, and a generic mechanism + for mapping between IPv6 addresses and IPv4 addresses and + transport layer ports."; + reference + "I-D.ietf-softwire-map"; + } + + grouping map-rule-table { + description + "The (conceptual) table containing rule Information for + a specific mapping rule. It can also be used for row creation."; + list map-rule-entry { + key "id"; + leaf id { + type uint8; + } + + leaf testbool { + type boolean; + } + } + } + }]] + + -- Convert the schema using the already tested parser. + local parser = require("lib.yang.parser") + local schema = parser.parse_string(test_schema, "schema selftest") + local mod = load_schema(schema) + + assert(mod.config.name == "ietf-softwire") + assert(mod.config.namespace == "urn:ietf:params:xml:ns:yang:ietf-softwire") + assert(mod.config.prefix == "softwire") + assert(mod.config.contact) + assert(mod.config.organization) + assert(mod.config.description) + + -- Check both modules exist. (Also need to check they've loaded) + assert(mod.modules["ietf-inet-types"]) + assert(mod.modules["ietf-yang-types"]) + + -- Check all revisions are accounted for. + assert(mod.revisions["2015-02-02"].config.description) + assert(mod.revisions["2015-02-06"].config.description) + assert(mod.revisions["2015-02-10"].config.description) + assert(mod.revisions["2015-04-07"].config.description) + assert(mod.revisions["2015-09-30"].config.description) + + -- Check that the feature statements are there. + assert(mod.features["lw4over6"].config.description) + assert(mod.features["lw4over6"].config.reference) + assert(mod.features["map-e"].config.description) + assert(mod.features["map-e"].config.reference) + + -- Check the grouping exists and has everything we'd want it to. + assert(mod.groupings["map-rule-table"].config.description) + assert(mod.groupings["map-rule-table"]["map-rule-entry"]) + + -- Check that the list was in the group was identified correctly. + local list = mod.groupings["map-rule-table"]["map-rule-entry"] + assert(list.config.key == "id") + assert(list.id.config.type == "uint8") + + -- Test both setting and getting ints and bools + list.id.box.value = 72 + assert(list.id.box.value == 72) + list.testbool.box.value = true + assert(list.testbool.box.value == true) + list.testbool.box.value = false + assert(list.testbool.box.value == false) + + -- Should fail. + list.testbool.box.value = "hello" +end diff --git a/src/lib/yang/validation.lua b/src/lib/yang/validation.lua new file mode 100644 index 0000000000..d76135888e --- /dev/null +++ b/src/lib/yang/validation.lua @@ -0,0 +1,34 @@ +module(..., package.seeall) + +function cardinality(kw, name, statements, haystack) + for s, c in pairs(statements) do + if (c[1] >= 1 and (not haystack[s])) or (#statements[s] < c[1] and #statements[s] > c[2]) then + if c[1] == c[2] then + error(("Expected %d %s statement(s) in %s:%s"):format( + c[1], s, kw, name)) + else + error( + ("Expected between %d and %d of %s statement(s) in %s:%s"):format( + c[1], c[2], s, kw, name)) + end + end + end + return true +end + +function validate_container(name, src) + local cardinality = {config={0,1}, description={0,1}, presense={0,1}, + reference={0,1}, status={0,1}, when={0,1}} + validate_cardinality("container", name, cardinality, src) +end + +function validate_uses(name, src) + local cardinality = {augment={0,1}, description={0,1}, refine={0,1}, + reference={0,1}, status={0,1}, when={0,1}} + validate_cardinality("uses", name, cardinality, src) +end + +function validate_notification(name, src) + local cardinality = {description={0,1}, refernece={0,1}, status={0,1}} + validate_cardinality("notification", name, cardinality, src) +end From 1b50533eb345630c2eca1161339c4bbfcf4b2358 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 10 May 2016 11:47:15 +0200 Subject: [PATCH 013/631] Improve schema and data tree builders --- src/lib/yang/helpers.lua | 79 +++++ src/lib/yang/schema.lua | 625 +++++++++++++++++++----------------- src/lib/yang/validation.lua | 23 +- src/lib/yang/yang.lua | 290 +++++++++++++++++ 4 files changed, 710 insertions(+), 307 deletions(-) create mode 100644 src/lib/yang/helpers.lua create mode 100644 src/lib/yang/yang.lua diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua new file mode 100644 index 0000000000..39156b5fd4 --- /dev/null +++ b/src/lib/yang/helpers.lua @@ -0,0 +1,79 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +function split(delimiter, text) + -- This is taken from syslog. + if delimiter == "" then return {text} end + if #text == 0 then return {} end + local list = {} + local pos = 1 + while true do + local first, last = text:find(delimiter, pos) + if first then + list[#list + 1] = text:sub(pos, first - 1) + pos = last + 1 + else + list[#list + 1] = text:sub(pos) + break + end + end + return list +end + +function extract_nodes(schema) + local nodes = {} + for _, v in pairs(schema) do + -- Recursively apply this. + if v.statements then v.statements = extract_nodes(v.statements) end + + -- Add to the nodes table. + if nodes[v.keyword] then + table.insert(nodes[v.keyword], v) + else + nodes[v.keyword] = {v} + end + end + return nodes +end + +Container = {} +function Container.new(base, path) + local ret = {root={}, base=base, path=path} + local meta = {} + return setmetatable(ret, { + __newindex = function (t, k, v) + -- Validate the value prior to setting it. + local schema = ret.base:get_schema(ret.path.."."..k) + if schema.validation then + for _, validation in pairs(schema.validation) do + validation(v) + end + end + + local box = ret.root[k] + if box == nil then error("Unknown leaf '"..k.."'") end + box.value = v + end, + + __index = function(t, k) + local table = rawget(t, "root") + local prop = table[k] + if not prop then + return prop + elseif prop.value == nil then + return prop + else + return prop.value + end + end}) +end + +function selftest() + local result = split("%.%.", "0..9") + assert(result[1] == "0") + assert(result[2] == "9") + + local result = split("%-", "hello-lua") + assert(result[1] == "hello") + assert(result[2] == "lua") +end \ No newline at end of file diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 8716bb30fe..7e283a07df 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -1,41 +1,17 @@ --- Built in types (page 19) --- --- +---------------------+-------------------------------------+ --- | Name | Description | --- +---------------------+-------------------------------------+ --- | binary | Any binary data | --- | bits | A set of bits or flags | --- | boolean | "true" or "false" | --- | decimal64 | 64-bit signed decimal number | --- | empty | A leaf that does not have any value | --- | enumeration | Enumerated strings | --- | identityref | A reference to an abstract identity | --- | instance-identifier | References a data tree node | --- | int8 | 8-bit signed integer | --- | int16 | 16-bit signed integer | --- | int32 | 32-bit signed integer | --- | int64 | 64-bit signed integer | --- | leafref | A reference to a leaf instance | --- | string | Human-readable string | --- | uint8 | 8-bit unsigned integer | --- | uint16 | 16-bit unsigned integer | --- | uint32 | 32-bit unsigned integer | --- | uint64 | 64-bit unsigned integer | --- | union | Choice of member types | --- +---------------------+-------------------------------------+ --- --- Module: --- module { ... } --- --- required: namespace, prefix --- optional: anyxml, augment, choice, contact, container, description, deviation, --- extension, feature, grouping, identity, import, include, leaf, leaf-list --- list, notification, organization, reference, revision, rpc, typedef, uses, --- yang-version +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +-- This module implements the schema tree and validation for YANG. It represents +-- the YANG statements with lua tables and provides a fast but flexible way to +-- represent and validate statements. +-- +-- Since YANG statements are encapsulated in modules at the highest level one +-- should take their pre-parsed YANG document containing the module and load it +-- into the Module table. -- +-- This relies on the "Base" table which can be found in the yang.lua file. module(..., package.seeall) local validation = require("lib.yang.validation") +local helpers = require("lib.yang.helpers") -- Use ffi types because they will validate that numeric values are being -- provided. The downside is that integer overflow could occur on these. This @@ -54,305 +30,345 @@ typedef struct { uint64_t value; } uint64box; typedef struct { double value; } decimal64box; ]] +local Leaf = {} +function Leaf.new(base, path, src) + self = setmetatable({}, {__index=Leaf, path=path}) + + -- Parse the schema to find the metadata + self:validate_schema(src) + base:add_cache(path, self) + + self.type = src.type[1].argument + if src.type[1].statements then + local typeinfo = src.type[1].statements + if typeinfo.range then + local range = helpers.split("%.%.", typeinfo.range[1].argument) + self.range = {tonumber(range[1]), tonumber(range[2])} + end + end + if src.description then + self.description = src.description[1].argument + end --- This is a boxed value for datatypes which are represeted in Lua and validated --- in Lua. This includes everything except the numeric values. -local Box = {} -function Box.new(validate) - local ret = {realroot={}} - local mt = { - __newindex = function (t, k, v) - if validate then validate(v) end - ret.realroot[k] = v - end, - __index = function(t, k) - local v = ret.realroot[k] - return v - end, - } - return setmetatable(ret, mt) -end --- Yang feature -local Feature = {} -function Feature.new(name) - local ret = {config={name=name}} - return setmetatable(ret, {__index = Feature}) -end - -function Feature:validate(statements) - local cardinality = {description={0,1}, status={0,1}, refernece={0,1}} - validation.cardinality("feature", name, cardinality, statements) -end - -function Feature:consume(statements) - if statements.description then - self.config.description = statements.description[1].argument + if src.default then + self.default = src.default[1].argument end - if statements.reference then - self.config.reference = statements.reference[1].argument + + if src["if-feature"] then + self["if-feature"] = {} + for _, f in pairs(src["if-feature"]) do + table.insert(self["if-feature"], f.argument) + end end - if statements.status then - self.config.status = statements.reference[1].argument + + if src.mandatory then + self.mandatory = src.mandatory[1].argument == "true" end + + -- Add validators if we need to. + if self.mandatory then + if not self.validation then self.validation = {} end + table.insert(self.validation, function (v) + if not v then + self:error("Value is mandatory") + end + end) + end + if self.range then + if not self.validation then self.validation = {} end + self.validation[#self.validation + 1] = function(v) + if v < self.range[1] or v > self.range[2] then + self:error("Value '%s' is out of range", path, value) + end + end + end + return self end --- Yang Leaf -local Leaf = {} -function Leaf.new(name) - local ret = {config={name=name}} - ret.config.iffeature = {} - return setmetatable(ret, {__index = Leaf}) +function Leaf:error(msg, ...) + local path = getmetatable(self).path + error(("%s: %s"):format(path, msg:format(...))) end -function Leaf:validate(statements) +function Leaf:validate_schema(schema) local cardinality = {config={0,1}, default={0,1}, description={0,1}, mandatory={0,1}, reference={0,1}, status={0,1}, type={1,1}, units={0,1}, when={0,1}} - validation.cardinality("leaf", name, cardinality, src) + validation.cardinality("leaf", getmetatable(self).path, cardinality, schema) end -function Leaf:consume(statements) - if statements.default then self.config.default = statements.default end - if statements.description then - self.config.description = statements.description[1].argument - end - - if statements["if-feature"] then - for _, f in pairs(statements["if-feature"]) do - table.insert(self.config.iffconfig["if-feature"], f.argument) - end - end - - -- Handle the type which will involve picking the correct box - -- to store the value with validation if needed. - self.config.type = statements.type[1].argument - - -- First deal with built in types. - if self.config.type == "int8" then - self.box = ffi.new("int8box") - elseif self.config.type == "int16" then - self.box = ffi.new("int16box") - elseif self.config.type == "int32" then - self.box = ffi.new("int32box") - elseif self.config.type == "int64" then - self.box = ffi.new("int64box") - elseif self.config.type == "uint8" then - self.box = ffi.new("uint8box") - elseif self.config.type == "uint16" then - self.box = ffi.new("uint16box") - elseif self.config.type == "uint32" then - self.box = ffi.new("uint32box") - elseif self.config.type == "uint64" then - self.box = ffi.new("uint64box") - elseif self.config.type == "decimal64" then - self.box = ffi.new("decimal64box") - elseif self.config.type == "string" then - self.box = Box.new() - elseif self.config.type == "boolean" then - self.box = Box.new(function (value) - if type(value) ~= "boolean" then - error( - ("Value for '%s' must be true or false."):format(self.config.name)) - end - end) +function Leaf:provide_box() + local box + + if self.type == "int8" then + box = ffi.new("int8box") + elseif self.type == "int16" then + box = ffi.new("int16box") + elseif self.type == "int32" then + box = ffi.new("int32box") + elseif self.type == "int64" then + box = ffi.new("int64box") + elseif self.type == "uint8" then + box = ffi.new("uint8box") + elseif self.type == "uint16" then + box = ffi.new("uint16box") + elseif self.type == "uint32" then + box = ffi.new("uint32box") + elseif self.type == "uint64" then + box = ffi.new("uint64box") + elseif self.type == "decimal64" then + box = ffi.new("decimal64box") + elseif self.type == "string" then + box = {} + elseif self.type == "boolean" then + box = {} + elseif self.type == "inet:" then + -- TODO: provide propper type. + box = {} else error(("Unknown type '%s' for leaf '%s'"):format( leaf_type, self.config.name)) end + + return box +end + +-- Yang feature +local Feature = {} +function Feature.new(base, path, src) + local self = setmetatable({}, {__index=Feature, path=path}) + + self:validate_schema(src) + base:add_cache(path, self) + + if src.description then + self.description = src.description[1].argument + end + if src.reference then + self.reference = src.reference[1].argument + end + if src.status then + self.status = src.reference[1].argument + end + + return self +end + +function Feature:validate_schema(src) + local cardinality = {description={0,1}, status={0,1}, refernece={0,1}} + validation.cardinality("feature", getmetatable(self).path, cardinality, src) end -- Yang list local List = {} -function List.new(name) - local ret = {config={name=name}} - return setmetatable(ret, {__index = List}) +function List.new(path, src) + local self = setmetatable({}, {__index=List, path=path}) + + self:validate_schema(src) + base:add_cache(path, self) + + if src.key then self.key = src.key[1].argument end + if src.leaf then + for _, leaf in pairs(src.leaf) do + local path = self.path.."."..leaf.argument + self[leaf.argument] = Leaf.new(base, path, leaf.statements) + end + end + + return self end -function List:validate(statements) +function List:validate_schema(src) local cardinality = {config={0,1}, description={0,1}, key={0,1}, reference={0,1}, status={0,1}, when={0,1}} cardinality["max-elements"] = {0,1} cardinality["min-elements"] = {0,1} cardinality["ordered-by"] = {0,1} - validation.cardinality("list", name, cardinality, statements) + validation.cardinality("list", getmetatable(self).path, cardinality, src) end -function List:consume(statements) - if statements.key then - self.config.key = statements.key[1].argument +-- Yang group +local Grouping = {} +function Grouping.new(base, path, src) + local ret = {leaves={}} + local self = setmetatable(ret, {__index = Grouping, path=path}) + + self:validate_schema(src) + base:add_cache(path, self) + + if src.description then + self.description = src.description[1].argument end - if statements.leaf then - for _, leaf in pairs(statements.leaf) do - self[leaf.argument] = Leaf.new(leaf.argument) - self[leaf.argument]:consume(leaf.statements) + + if src.list then + for _, list in pairs(src.list) do + local path = path.."."..list.argument + self[list.argument] = List.new(base, path, list.statements) end end -end --- Yang group -local Grouping = {} -function Grouping.new(name) - local ret = {config={name=name}} - return setmetatable(ret, {__index = Grouping}) + if src.leaf then + for _, leaf in pairs(src.leaf) do + local path = path.."."..leaf.argument + self.leaves[leaf.argument] = Leaf.new(base, path, leaf.statements) + end + end + + return self end -function Grouping:validate(statements) +function Grouping:validate_schema(src) local cardinality = {description={0,1}, reference={0,1}, status={0,1}} - validation.cardinality("grouping", name, cardinality, statements) + validation.cardinality("grouping", getmetatable(self).path, cardinality, src) end -function Grouping:consume(statements) - if statements.description then - self.config.description = statements.description[1].argument - end +local Container = {} +function Container.new(base, path, src) + local ret = {leaves={}} + local self = setmetatable(ret, {__index=Container, path=path}) - if statements.list then - for _, list in pairs(statements.list) do - self[list.argument] = List.new(list.argument) - self[list.argument]:consume(list.statements) - end + self:validate_schema(src) + base:add_cache(path, self) + + if src.description then + self.description = src.description[1].argument end - if statements.leaf then - for _, leaf in pairs(statements.leaf) do - self[leaf.argument] = Leaf.new(leaf.argument) - self[leaf.argument]:consume(leaf.statements) + -- Leaf statements + if src.leaf then + for _, leaf in pairs(src.leaf) do + local leaf_path = path.."."..leaf.argument + self.leaves[leaf.argument] = Leaf.new(base, leaf_path, leaf.statements) end end + + return self +end + +function Container:validate_schema(src) + local cardinality = {config={0,1}, description={0,1}, presense={0,1}, + reference={0,1}, status={0,1}, when={0,1}} + validation.cardinality("container", getmetatable(self).path, cardinality, src) end -- Yang Revision local Revision = {} -function Revision.new(date) - local ret = {config={date=date}} - return setmetatable(ret, {__index = Revision}) -end +function Revision.new(base, path, src) + local self = setmetatable({}, {__index=Revision, path=path}) -function Revision:validate(statements) - local cardinality = {description={0,1}, reference={0,1}} - validation.cardinality("revision", self.config.date, cardinality, statements) -end + self:validate_schema(src) -function Revision:consume(statements) - self:validate(statements) + base:add_cache(path, self) - if statements.description then - self.config.description = statements.description[1].argument + if src.description then + self.description = src.description[1].argument end - if statements.description then - self.config.reference = statements.reference.argument + if src.description then + self.reference = src.reference[1].argument end -end --- Yang Module -local Module = {} -function Module.new(name) - local ret = {body={}, config={name=name}, modules={}, revisions={}, - features={}, groupings={}} - return setmetatable(ret, {__index = Module}) + return self end -function Module:load() - -- Find the file and load it. - print("DEBUG: loading module", self.config.name) +function Revision:validate_schema(src) + local cardinality = {description={0,1}, reference={0,1}} + validation.cardinality("revision", getmetatable(self).path, cardinality, src) end -function Module:consume(statements) - -- Validate the statements first. - self:validate(statements) +-- Yang Module +Module = {} +function Module.new(base, name, src) + local ret = {body={}, name=name, modules={}, revisions={}, + features={}, groupings={}, containers={}} + local self = setmetatable(ret, {__index=Module, path=name}) + + -- TODO: remove me when proper loading support exists. + if not src then return self end + + -- Add self to path cache + base:add_cache(name, self) + + -- Validate the statements first. + self:validate_schema(src) -- Set the meta information about the module - self.config.namespace = statements.namespace[1].argument - self.config.prefix = statements.prefix[1].argument + self.namespace = src.namespace[1].argument + self.prefix = src.prefix[1].argument - if statements.organization then - self.config.organization = statements.organization[1].argument + if src.organization then + self.organization = src.organization[1].argument end - if statements.contact then - self.config.contact = statements.contact[1].argument + if src.contact then + self.contact = src.contact[1].argument end - if statements.description then - self.config.description = statements.description[1].argument + if src.description then + self.description = src.description[1].argument end -- Now handle the imports, as other things may reference them. - if statements.import then - for _, mod in pairs(statements.import) do - self.modules[mod.argument] = Module.new(mod.argument) + if src.import then + for _, mod in pairs(src.import) do + self.modules[mod.argument] = Module.new(base, mod.argument) -- Ask the module to find and load itself. self.modules[mod.argument]:load() + + -- TODO Add module to path cache. end end -- Handle revisions - if statements.revision then - for _, revision in pairs(statements.revision) do - -- TODO: can or should we convert these revision date stamps. - self.revisions[revision.argument] = Revision.new(revision.argument) - self.revisions[revision.argument]:consume(revision.statements) + if src.revision then + for _, r in pairs(src.revision) do + local path = ret.name.."."..r.argument + self.revisions[r.argument] = Revision.new(base, path, r.statements) end end -- Feature statements - if statements.feature then - for _, feature in pairs(statements.feature) do - self.features[feature.argument] = Feature.new(feature.argument) - self.features[feature.argument]:consume(feature.statements) + if src.feature then + for _, f in pairs(src.feature) do + local path = ret.name.."."..f.argument + self.features[f.argument] = Feature.new(base, path, f.statements) end end -- Leaf statements - if statements.leaf then - for _, leaf in pairs(statements.revision) do + if src.leaf then + for _, leaf in pairs(src.leaf) do end end -- List statements - if statements.grouping then - for _, grouping in pairs(statements.grouping) do - self.groupings[grouping.argument] = Grouping.new(grouping.argument) - self.groupings[grouping.argument]:consume(grouping.statements) + if src.grouping then + for _, g in pairs(src.grouping) do + local path = ret.name.."."..g.argument + self.groupings[g.argument] = Grouping.new(base, path, g.statements) end end -end -function Module:validate(statements) - local cardinality = {contact={0,1}, description={0,1}, namespace={1,1}, - organization={0,1}, prefix={1,1}, reference={0,1}} - cardinality["yang-version"] = {0,1} - validation.cardinality("module", self.name, cardinality, statements) -end - - -function extract_nodes(structure) - local nodes = {} - for _, v in pairs(structure) do - -- Recursively apply this. - if v.statements then v.statements = extract_nodes(v.statements) end - - -- Add to the nodes table. - if nodes[v.keyword] then - table.insert(nodes[v.keyword], v) - else - nodes[v.keyword] = {v} + -- Containers + if src.container then + for _, c in pairs(src.container) do + local path = ret.name.."."..c.argument + self.containers[c.argument] = Container.new(base, path, c.statements) end end - return nodes -end -function load_schema(src) - -- Okay conver the schema into something more useful. - src = extract_nodes(src) + return self +end - -- Okay we're expecting a module at the top - if not src.module then error("Expected 'module'") end - local mod = Module.new(src.module[1].argument) - mod:consume(src.module[1].statements) - return mod +function Module:load() + -- TODO: Find the file and load it. end +function Module:validate_schema(src) + local cardinality = {contact={0,1}, description={0,1}, namespace={1,1}, + organization={0,1}, prefix={1,1}, reference={0,1}} + cardinality["yang-version"] = {0,1} + validation.cardinality("module", getmetatable(self).path, cardinality, src) +end function selftest() local test_schema = [[module ietf-softwire { @@ -446,69 +462,104 @@ function selftest() "I-D.ietf-softwire-map"; } - grouping map-rule-table { + grouping port-set { + description + "Use the PSID algorithm to represent a range of transport layer + ports."; + leaf offset { + type uint8 { + range 0..16; + } + mandatory true; + description + "The number of offset bits. In Lightweight 4over6, the defaul + value is 0 for assigning one contiguous port range. In MAP-E/T, + the default value is 6, which excludes system ports by default + and assigns distributed port ranges. If the this parameter is + larger than 0, the value of offset MUST be greater than 0."; + } + leaf psid { + type uint16; + mandatory true; description - "The (conceptual) table containing rule Information for - a specific mapping rule. It can also be used for row creation."; - list map-rule-entry { - key "id"; - leaf id { - type uint8; - } - - leaf testbool { - type boolean; - } + "Port Set Identifier (PSID) value, which identifies a set + of ports algorithmically."; + } + leaf psid-len { + type uint8 { + range 0..16; } + mandatory true; + description + "The length of PSID, representing the sharing ratio for an + IPv4 address."; } + } + + container softwire-config { + description + "The configuration data for Softwire instances. And the shared + data describes the softwire data model which is common to all of + the different softwire mechanisms, such as description."; + leaf description { + type string; + description + "A textual description of Softwire."; + } + } }]] -- Convert the schema using the already tested parser. local parser = require("lib.yang.parser") local schema = parser.parse_string(test_schema, "schema selftest") - local mod = load_schema(schema) - assert(mod.config.name == "ietf-softwire") - assert(mod.config.namespace == "urn:ietf:params:xml:ns:yang:ietf-softwire") - assert(mod.config.prefix == "softwire") - assert(mod.config.contact) - assert(mod.config.organization) - assert(mod.config.description) + -- Create a fake base, we're not testing this so avoidng using the real one. + local base = {add_cache = function() end} + + -- Convert the schema into a more usable form for us. + schema = helpers.extract_nodes(schema) + + -- Load the module + local mod = Module.new(base, schema.module[1].argument, + schema.module[1].statements) + + print("module: ", mod.name) + assert(mod.name == "ietf-softwire") + assert(mod.namespace == "urn:ietf:params:xml:ns:yang:ietf-softwire") + assert(mod.prefix == "softwire") + assert(mod.contact) + assert(mod.organization) + assert(mod.description) -- Check both modules exist. (Also need to check they've loaded) assert(mod.modules["ietf-inet-types"]) assert(mod.modules["ietf-yang-types"]) -- Check all revisions are accounted for. - assert(mod.revisions["2015-02-02"].config.description) - assert(mod.revisions["2015-02-06"].config.description) - assert(mod.revisions["2015-02-10"].config.description) - assert(mod.revisions["2015-04-07"].config.description) - assert(mod.revisions["2015-09-30"].config.description) + assert(mod.revisions["2015-02-02"].description) + assert(mod.revisions["2015-02-06"].description) + assert(mod.revisions["2015-02-10"].description) + assert(mod.revisions["2015-04-07"].description) + assert(mod.revisions["2015-09-30"].description) -- Check that the feature statements are there. - assert(mod.features["lw4over6"].config.description) - assert(mod.features["lw4over6"].config.reference) - assert(mod.features["map-e"].config.description) - assert(mod.features["map-e"].config.reference) - - -- Check the grouping exists and has everything we'd want it to. - assert(mod.groupings["map-rule-table"].config.description) - assert(mod.groupings["map-rule-table"]["map-rule-entry"]) - - -- Check that the list was in the group was identified correctly. - local list = mod.groupings["map-rule-table"]["map-rule-entry"] - assert(list.config.key == "id") - assert(list.id.config.type == "uint8") - - -- Test both setting and getting ints and bools - list.id.box.value = 72 - assert(list.id.box.value == 72) - list.testbool.box.value = true - assert(list.testbool.box.value == true) - list.testbool.box.value = false - assert(list.testbool.box.value == false) - - -- Should fail. - list.testbool.box.value = "hello" -end + assert(mod.features["lw4over6"].description) + assert(mod.features["lw4over6"].reference) + assert(mod.features["map-e"].description) + assert(mod.features["map-e"].reference) + + -- Check the groupings + assert(mod.groupings["port-set"]) + assert(mod.groupings["port-set"].description) + assert(mod.groupings["port-set"].leaves["offset"]) + assert(mod.groupings["port-set"].leaves["offset"].type == "uint8") + assert(mod.groupings["port-set"].leaves["psid-len"].mandatory == true) + assert(mod.groupings["port-set"].leaves["psid-len"].range[1] == 0) + assert(mod.groupings["port-set"].leaves["psid-len"].range[2] == 16) + + -- Check the containers description (NOT the leaf called "description") + assert(type(mod.containers["softwire-config"].description) == "string") + + -- Check the container has a leaf called "description" + assert(mod.containers["softwire-config"].leaves.description) +end \ No newline at end of file diff --git a/src/lib/yang/validation.lua b/src/lib/yang/validation.lua index d76135888e..84847ee41d 100644 --- a/src/lib/yang/validation.lua +++ b/src/lib/yang/validation.lua @@ -1,34 +1,17 @@ module(..., package.seeall) -function cardinality(kw, name, statements, haystack) +function cardinality(kw, path, statements, haystack) for s, c in pairs(statements) do if (c[1] >= 1 and (not haystack[s])) or (#statements[s] < c[1] and #statements[s] > c[2]) then if c[1] == c[2] then error(("Expected %d %s statement(s) in %s:%s"):format( - c[1], s, kw, name)) + c[1], s, kw, path)) else error( ("Expected between %d and %d of %s statement(s) in %s:%s"):format( - c[1], c[2], s, kw, name)) + c[1], c[2], s, kw, path)) end end end return true end - -function validate_container(name, src) - local cardinality = {config={0,1}, description={0,1}, presense={0,1}, - reference={0,1}, status={0,1}, when={0,1}} - validate_cardinality("container", name, cardinality, src) -end - -function validate_uses(name, src) - local cardinality = {augment={0,1}, description={0,1}, refine={0,1}, - reference={0,1}, status={0,1}, when={0,1}} - validate_cardinality("uses", name, cardinality, src) -end - -function validate_notification(name, src) - local cardinality = {description={0,1}, refernece={0,1}, status={0,1}} - validate_cardinality("notification", name, cardinality, src) -end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua new file mode 100644 index 0000000000..989721b248 --- /dev/null +++ b/src/lib/yang/yang.lua @@ -0,0 +1,290 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +-- This is the entry-point to the lua YANG library. It will handle parsing a +-- file with YANG statements and producing a schema tree and data tree. +-- +-- To use this you should use the helper functions: +-- - load_schema +-- - load_schema_file +-- The former takes in a yang string and builds a Base table, similiaraly the +-- load_schema_file takes a schema file and builds a Base table from that. +-- +-- Once you have the base file you can inspect the schema, data and set values. +-- Examaple: +-- local base = load_schema[[ +-- module example { +-- namespace "exampleinc:example"; +-- prefix "example"; +-- organization "Example Inc."; +-- description "An example document" +-- +-- container conf { +-- leaf description { +-- type string; +-- } +-- leaf testvalue { +-- type uint8; +-- } +-- } +-- }]] +-- base["example"].organization == "YANG IPv4 over IPv6" +-- base["example"].containers["conf"].leaves["description"].type == "string" +-- +-- -- Setting data +-- base.data["example"]["conf"].description = "hello!" +-- base.data["example"]["conf"].testvalue = "error" -- This errors. +-- base.data["example"]["conf"].testvalue = 50000 -- Beware: they overflow. +module(..., package.seeall) + +local schema = require("lib.yang.schema") +local helpers = require("lib.yang.helpers") +local parser = require("lib.yang.parser") + +local Base = {} +function Base.new(filename) + ret = {schema={}, filename=filename} + self = setmetatable(ret, {__index=Base, path_cache={}}) + self.data = helpers.Container.new(self, "") + return self +end + +function Base:error(path, node, msg, ...) + error(("Error: %s.%s: %s"):format(path, node, (msg):format(...))) +end + +function Base:load(src) + src = helpers.extract_nodes(src) + if not src.module then + error(("%s: Expected 'module'"):format(self.filename)) + end + local mod = schema.Module.new( + self, + src.module[1].argument, + src.module[1].statements + ) + + self.schema[src.module[1].argument] = mod + -- TODO: don't use rawget here. + local data_root = rawget(self.data, "root") + data_root[mod.name] = helpers.Container.new(self, mod.name) + return self.schema +end + +function Base:get_schema(path) + -- Handle case when path is empty. (Provide with self.data) + if path == "" then + return self.data + end + + -- Look in the cache and hopefully return the schema node for the path. + local cache = getmetatable(self).path_cache + return cache[path] +end + +function Base:add_cache(path, node) + getmetatable(self).path_cache[path] = node +end + +function Base:produce_data_tree(schema_node, data_node) + if not (schema_node and data_node) then + for _, mod in pairs(self.schema) do + schema_node = mod + end + if not schema_node then error("Module cannot be resolved") end + data_node = self.data[schema_node.name] + end + path = getmetatable(schema_node).path + + if schema_node.containers then + for name, container in pairs(schema_node.containers) do + local new_path = path.."."..name + local new_node = helpers.Container.new(self, new_path) + -- TODO: change me, we shouldn't be using rawget here! + local root = rawget(data_node, "root") + root[name] = new_node + schema_node = self:get_schema(new_path) + self:produce_data_tree(schema_node, new_node) + end + end + + if schema_node.leaves then + for name, leaf in pairs(schema_node.leaves) do + -- TODO remove the use of rawget here, it's not good. + local root = rawget(data_node, "root") + root[name] = leaf:provide_box() + end + end + + return self.data +end + +function Base:add(key, node) + self.data[key] = node +end + +function load_schema(schema, filename) + local parsed_yang = parser.parse_string(schema, "selftest") + local base = Base.new() + base:load(parsed_yang) + return base +end + +function load_schema_file(filename) + local file_in = assert(io.open(filename)) + local contents = file_in:read("*a") + return load_schema(contents, filename) +end + +function selftest() + local test_yang = [[module ietf-softwire { + namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; + prefix "softwire"; + + import ietf-inet-types {prefix inet; } + import ietf-yang-types {prefix yang; } + + organization "Softwire Working Group"; + + contact + " + Qi Sun sunqi.ietf@gmail.com + Hao Wang wangh13@mails.tsinghua.edu.cn + Yong Cui yong@csnet1.cs.tsinghua.edu.cn + Ian Farrer ian.farrer@telekom.de + Mohamed Boucadair mohamed.boucadair@orange.com + Rajiv Asati rajiva@cisco.com + "; + + description + "This document defines a YANG data model for the configuration and + management of IPv4-in-IPv6 Softwire Border Routers and Customer + Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T + Softwire mechanisms. + + Copyright (c) 2014 IETF Trust and the persons identified + as authors of the code. All rights reserved. + This version of this YANG module is part of RFC XXX; see the RFC + itself for full legal notices."; + + + revision 2015-09-30 { + description + "Version-04: fix YANG syntax; Add flags to map-rule; Remove + the map-rule-type element. "; + reference "tbc"; + } + + revision 2015-04-07 { + description + "Version-03: Integrate lw4over6; Updata state nodes; Correct + grammar errors; Reuse groupings; Update descriptions. + Simplify the model."; + reference "tbc"; + } + + revision 2015-02-10 { + description + "Version-02: Add notifications."; + reference "tbc"; + } + + + revision 2015-02-06 { + description + "Version-01: Correct grammar errors; Reuse groupings; Update + descriptions."; + reference "tbc"; + } + + revision 2015-02-02 { + description + "Initial revision."; + reference "tbc"; + } + + feature lw4over6 { + description + "Lightweight 4over6 moves the Network Address and Port + Translation (NAPT) function from the centralized DS-Lite tunnel + concentrator to the tunnel client located in the Customer + Premises Equipment (CPE). This removes the requirement for a + Carrier Grade NAT function in the tunnel concentrator and + reduces the amount of centralized state that must be held to a + per-subscriber level. In order to delegate the NAPT function + and make IPv4 Address sharing possible, port-restricted IPv4 + addresses are allocated to the CPEs."; + reference + "I-D.ietf-softwire-lw4over6"; + } + + feature map-e { + description + "MAP-E is a mechanism for transporting IPv4 packets across an + IPv6 network using IP encapsulation, and a generic mechanism + for mapping between IPv6 addresses and IPv4 addresses and + transport layer ports."; + reference + "I-D.ietf-softwire-map"; + } + + grouping port-set { + description + "Use the PSID algorithm to represent a range of transport layer + ports."; + leaf offset { + type uint8 { + range 0..16; + } + mandatory true; + description + "The number of offset bits. In Lightweight 4over6, the defaul + value is 0 for assigning one contiguous port range. In MAP-E/T, + the default value is 6, which excludes system ports by default + and assigns distributed port ranges. If the this parameter is + larger than 0, the value of offset MUST be greater than 0."; + } + leaf psid { + type uint16; + mandatory true; + description + "Port Set Identifier (PSID) value, which identifies a set + of ports algorithmically."; + } + leaf psid-len { + type uint8 { + range 0..16; + } + mandatory true; + description + "The length of PSID, representing the sharing ratio for an + IPv4 address."; + } + } + + container softwire-config { + description + "The configuration data for Softwire instances. And the shared + data describes the softwire data model which is common to all of + the different softwire mechanisms, such as description."; + leaf description { + type string; + description + "A textual description of Softwire."; + } + + leaf testvalue { + type uint8; + description "Test value for unsigned 8 bit integers."; + } + } +}]] + local base = load_schema(test_yang) + local data = base:produce_data_tree() + + -- Set the description leaf + data["ietf-softwire"]["softwire-config"].description = "I am a description" + assert(data["ietf-softwire"]["softwire-config"].description == "I am a description") + + -- Set the testvalue + data["ietf-softwire"]["softwire-config"].testvalue = 72 + assert(data["ietf-softwire"]["softwire-config"].testvalue == 72) +end \ No newline at end of file From 23f106009b2c290de2f2bad4f4b1c2894b41e000 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 10 May 2016 14:57:21 +0200 Subject: [PATCH 014/631] Fix indentation and few stylistic things --- src/lib/yang/helpers.lua | 120 +++---- src/lib/yang/parser.lua | 33 +- src/lib/yang/schema.lua | 738 +++++++++++++++++++-------------------- src/lib/yang/yang.lua | 402 ++++++++++----------- 4 files changed, 648 insertions(+), 645 deletions(-) diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index 39156b5fd4..7b8d686c28 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -2,78 +2,78 @@ module(..., package.seeall) function split(delimiter, text) - -- This is taken from syslog. - if delimiter == "" then return {text} end - if #text == 0 then return {} end - local list = {} - local pos = 1 - while true do - local first, last = text:find(delimiter, pos) - if first then - list[#list + 1] = text:sub(pos, first - 1) - pos = last + 1 - else - list[#list + 1] = text:sub(pos) - break - end - end - return list + -- This is taken from syslog. + if delimiter == "" then return {text} end + if #text == 0 then return {} end + local list = {} + local pos = 1 + while true do + local first, last = text:find(delimiter, pos) + if first then + list[#list + 1] = text:sub(pos, first - 1) + pos = last + 1 + else + list[#list + 1] = text:sub(pos) + break + end + end + return list end function extract_nodes(schema) - local nodes = {} - for _, v in pairs(schema) do - -- Recursively apply this. - if v.statements then v.statements = extract_nodes(v.statements) end + local nodes = {} + for _, v in pairs(schema) do + -- Recursively apply this. + if v.statements then v.statements = extract_nodes(v.statements) end - -- Add to the nodes table. - if nodes[v.keyword] then - table.insert(nodes[v.keyword], v) - else - nodes[v.keyword] = {v} - end - end - return nodes + -- Add to the nodes table. + if nodes[v.keyword] then + table.insert(nodes[v.keyword], v) + else + nodes[v.keyword] = {v} + end + end + return nodes end Container = {} function Container.new(base, path) - local ret = {root={}, base=base, path=path} - local meta = {} - return setmetatable(ret, { - __newindex = function (t, k, v) - -- Validate the value prior to setting it. - local schema = ret.base:get_schema(ret.path.."."..k) - if schema.validation then - for _, validation in pairs(schema.validation) do - validation(v) - end - end + local ret = {root={}, base=base, path=path} + local meta = {} + return setmetatable(ret, { + __newindex = function (t, k, v) + -- Validate the value prior to setting it. + local schema = ret.base:get_schema(ret.path.."."..k) + if schema.validation then + for _, validation in pairs(schema.validation) do + validation(v) + end + end - local box = ret.root[k] - if box == nil then error("Unknown leaf '"..k.."'") end - box.value = v - end, - - __index = function(t, k) - local table = rawget(t, "root") - local prop = table[k] - if not prop then - return prop - elseif prop.value == nil then - return prop - else - return prop.value + local box = ret.root[k] + if box == nil then error("Unknown leaf '"..k.."'") end + box.value = v + end, + __index = function(t, k) + local table = rawget(t, "root") + local prop = table[k] + if not prop then + return prop + elseif prop.value == nil then + return prop + else + return prop.value + end end - end}) + }) end function selftest() - local result = split("%.%.", "0..9") - assert(result[1] == "0") - assert(result[2] == "9") + local result = split("%.%.", "0..9") + assert(result[1] == "0") + assert(result[2] == "9") - local result = split("%-", "hello-lua") - assert(result[1] == "hello") - assert(result[2] == "lua") + local result = split("%-", "hello-lua") + assert(result[1] == "hello") + assert(result[2] == "lua") end \ No newline at end of file diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 1f44cff976..a8d4c5074f 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -277,6 +277,7 @@ end function parse_file(filename) local file_in = assert(io.open(filename)) local contents = file_in:read("*a") + file_in:close() return parse_string(contents, filename) end @@ -296,21 +297,21 @@ function selftest() local function pp(x) if type(x) == "table" then - io.write("{") - local first = true - for k,v in pairs(x) do - if not first then - io.write(", ") - end - io.write(k.."=") - pp(v) - first = false - end - io.write("}") + io.write("{") + local first = true + for k,v in pairs(x) do + if not first then + io.write(", ") + end + io.write(k.."=") + pp(v) + first = false + end + io.write("}") elseif type(x) == "string" then - io.write(x) + io.write(x) else - error("Unsupported type") + error("Unsupported type") end end @@ -318,9 +319,9 @@ function selftest() local function test_module(src, exp) local result = parse_string(src) if not lib.equal(result, exp) then - pp(result) - pp(exp) - error("no equal") + pp(result) + pp(exp) + error("no equal") end end diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 7e283a07df..e8aecd0cb4 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -32,38 +32,38 @@ typedef struct { double value; } decimal64box; local Leaf = {} function Leaf.new(base, path, src) - self = setmetatable({}, {__index=Leaf, path=path}) - - -- Parse the schema to find the metadata - self:validate_schema(src) - base:add_cache(path, self) - - self.type = src.type[1].argument - if src.type[1].statements then - local typeinfo = src.type[1].statements - if typeinfo.range then - local range = helpers.split("%.%.", typeinfo.range[1].argument) - self.range = {tonumber(range[1]), tonumber(range[2])} - end - end - if src.description then - self.description = src.description[1].argument - end - - if src.default then - self.default = src.default[1].argument - end - - if src["if-feature"] then - self["if-feature"] = {} - for _, f in pairs(src["if-feature"]) do - table.insert(self["if-feature"], f.argument) - end - end - - if src.mandatory then - self.mandatory = src.mandatory[1].argument == "true" - end + self = setmetatable({}, {__index=Leaf, path=path}) + + -- Parse the schema to find the metadata + self:validate_schema(src) + base:add_cache(path, self) + + self.type = src.type[1].argument + if src.type[1].statements then + local typeinfo = src.type[1].statements + if typeinfo.range then + local range = helpers.split("%.%.", typeinfo.range[1].argument) + self.range = {tonumber(range[1]), tonumber(range[2])} + end + end + if src.description then + self.description = src.description[1].argument + end + + if src.default then + self.default = src.default[1].argument + end + + if src["if-feature"] then + self["if-feature"] = {} + for _, f in pairs(src["if-feature"]) do + table.insert(self["if-feature"], f.argument) + end + end + + if src.mandatory then + self.mandatory = src.mandatory[1].argument == "true" + end -- Add validators if we need to. if self.mandatory then @@ -86,427 +86,425 @@ function Leaf.new(base, path, src) end function Leaf:error(msg, ...) - local path = getmetatable(self).path + local path = getmetatable(self).path error(("%s: %s"):format(path, msg:format(...))) end function Leaf:validate_schema(schema) - local cardinality = {config={0,1}, default={0,1}, description={0,1}, - mandatory={0,1}, reference={0,1}, status={0,1}, - type={1,1}, units={0,1}, when={0,1}} - validation.cardinality("leaf", getmetatable(self).path, cardinality, schema) + local cardinality = {config={0,1}, default={0,1}, description={0,1}, + mandatory={0,1}, reference={0,1}, status={0,1}, + type={1,1}, units={0,1}, when={0,1}} + validation.cardinality("leaf", getmetatable(self).path, cardinality, schema) end function Leaf:provide_box() - local box - - if self.type == "int8" then - box = ffi.new("int8box") - elseif self.type == "int16" then - box = ffi.new("int16box") - elseif self.type == "int32" then - box = ffi.new("int32box") - elseif self.type == "int64" then - box = ffi.new("int64box") - elseif self.type == "uint8" then - box = ffi.new("uint8box") - elseif self.type == "uint16" then - box = ffi.new("uint16box") - elseif self.type == "uint32" then - box = ffi.new("uint32box") - elseif self.type == "uint64" then - box = ffi.new("uint64box") - elseif self.type == "decimal64" then - box = ffi.new("decimal64box") - elseif self.type == "string" then - box = {} - elseif self.type == "boolean" then - box = {} - elseif self.type == "inet:" then - -- TODO: provide propper type. + local box + + if self.type == "int8" then + box = ffi.new("int8box") + elseif self.type == "int16" then + box = ffi.new("int16box") + elseif self.type == "int32" then + box = ffi.new("int32box") + elseif self.type == "int64" then + box = ffi.new("int64box") + elseif self.type == "uint8" then + box = ffi.new("uint8box") + elseif self.type == "uint16" then + box = ffi.new("uint16box") + elseif self.type == "uint32" then + box = ffi.new("uint32box") + elseif self.type == "uint64" then + box = ffi.new("uint64box") + elseif self.type == "decimal64" then + box = ffi.new("decimal64box") + elseif self.type == "string" then + box = {} + elseif self.type == "boolean" then box = {} - else - error(("Unknown type '%s' for leaf '%s'"):format( - leaf_type, self.config.name)) - end + else + error(("Unknown type '%s' for leaf '%s'"):format( + leaf_type, self.config.name)) + end - return box + return box end -- Yang feature local Feature = {} function Feature.new(base, path, src) - local self = setmetatable({}, {__index=Feature, path=path}) - - self:validate_schema(src) - base:add_cache(path, self) - - if src.description then - self.description = src.description[1].argument - end - if src.reference then - self.reference = src.reference[1].argument - end - if src.status then - self.status = src.reference[1].argument - end - - return self + local self = setmetatable({}, {__index=Feature, path=path}) + + self:validate_schema(src) + base:add_cache(path, self) + + if src.description then + self.description = src.description[1].argument + end + + if src.reference then + self.reference = src.reference[1].argument + end + + if src.status then + self.status = src.reference[1].argument + end + + return self end function Feature:validate_schema(src) - local cardinality = {description={0,1}, status={0,1}, refernece={0,1}} - validation.cardinality("feature", getmetatable(self).path, cardinality, src) + local cardinality = {description={0,1}, status={0,1}, refernece={0,1}} + validation.cardinality("feature", getmetatable(self).path, cardinality, src) end -- Yang list local List = {} function List.new(path, src) - local self = setmetatable({}, {__index=List, path=path}) + local self = setmetatable({}, {__index=List, path=path}) - self:validate_schema(src) - base:add_cache(path, self) + self:validate_schema(src) + base:add_cache(path, self) - if src.key then self.key = src.key[1].argument end - if src.leaf then - for _, leaf in pairs(src.leaf) do - local path = self.path.."."..leaf.argument - self[leaf.argument] = Leaf.new(base, path, leaf.statements) - end - end + if src.key then self.key = src.key[1].argument end + if src.leaf then + for _, leaf in pairs(src.leaf) do + local path = self.path.."."..leaf.argument + self[leaf.argument] = Leaf.new(base, path, leaf.statements) + end + end - return self + return self end function List:validate_schema(src) - local cardinality = {config={0,1}, description={0,1}, key={0,1}, - reference={0,1}, status={0,1}, when={0,1}} - cardinality["max-elements"] = {0,1} - cardinality["min-elements"] = {0,1} - cardinality["ordered-by"] = {0,1} - validation.cardinality("list", getmetatable(self).path, cardinality, src) + local cardinality = {config={0,1}, description={0,1}, key={0,1}, + reference={0,1}, status={0,1}, when={0,1}} + cardinality["max-elements"] = {0,1} + cardinality["min-elements"] = {0,1} + cardinality["ordered-by"] = {0,1} + validation.cardinality("list", getmetatable(self).path, cardinality, src) end -- Yang group local Grouping = {} function Grouping.new(base, path, src) - local ret = {leaves={}} - local self = setmetatable(ret, {__index = Grouping, path=path}) - - self:validate_schema(src) - base:add_cache(path, self) - - if src.description then - self.description = src.description[1].argument - end - - if src.list then - for _, list in pairs(src.list) do - local path = path.."."..list.argument - self[list.argument] = List.new(base, path, list.statements) - end - end - - if src.leaf then - for _, leaf in pairs(src.leaf) do - local path = path.."."..leaf.argument - self.leaves[leaf.argument] = Leaf.new(base, path, leaf.statements) - end - end - - return self + local ret = {leaves={}} + local self = setmetatable(ret, {__index = Grouping, path=path}) + + self:validate_schema(src) + base:add_cache(path, self) + + if src.description then + self.description = src.description[1].argument + end + + if src.list then + for _, list in pairs(src.list) do + local path = path.."."..list.argument + self[list.argument] = List.new(base, path, list.statements) + end + end + + if src.leaf then + for _, leaf in pairs(src.leaf) do + local path = path.."."..leaf.argument + self.leaves[leaf.argument] = Leaf.new(base, path, leaf.statements) + end + end + + return self end function Grouping:validate_schema(src) - local cardinality = {description={0,1}, reference={0,1}, status={0,1}} - validation.cardinality("grouping", getmetatable(self).path, cardinality, src) + local cardinality = {description={0,1}, reference={0,1}, status={0,1}} + validation.cardinality("grouping", getmetatable(self).path, cardinality, src) end local Container = {} function Container.new(base, path, src) - local ret = {leaves={}} - local self = setmetatable(ret, {__index=Container, path=path}) + local ret = {leaves={}} + local self = setmetatable(ret, {__index=Container, path=path}) - self:validate_schema(src) - base:add_cache(path, self) + self:validate_schema(src) + base:add_cache(path, self) - if src.description then - self.description = src.description[1].argument - end + if src.description then + self.description = src.description[1].argument + end - -- Leaf statements - if src.leaf then - for _, leaf in pairs(src.leaf) do - local leaf_path = path.."."..leaf.argument - self.leaves[leaf.argument] = Leaf.new(base, leaf_path, leaf.statements) - end - end + -- Leaf statements + if src.leaf then + for _, leaf in pairs(src.leaf) do + local leaf_path = path.."."..leaf.argument + self.leaves[leaf.argument] = Leaf.new(base, leaf_path, leaf.statements) + end + end - return self + return self end function Container:validate_schema(src) - local cardinality = {config={0,1}, description={0,1}, presense={0,1}, - reference={0,1}, status={0,1}, when={0,1}} - validation.cardinality("container", getmetatable(self).path, cardinality, src) + local cardinality = {config={0,1}, description={0,1}, presense={0,1}, + reference={0,1}, status={0,1}, when={0,1}} + validation.cardinality("container", getmetatable(self).path, cardinality, src) end -- Yang Revision local Revision = {} function Revision.new(base, path, src) - local self = setmetatable({}, {__index=Revision, path=path}) + local self = setmetatable({}, {__index=Revision, path=path}) - self:validate_schema(src) + self:validate_schema(src) + base:add_cache(path, self) - base:add_cache(path, self) - - if src.description then - self.description = src.description[1].argument - end - if src.description then - self.reference = src.reference[1].argument - end + if src.description then + self.description = src.description[1].argument + end - return self + if src.description then + self.reference = src.reference[1].argument + end + return self end function Revision:validate_schema(src) - local cardinality = {description={0,1}, reference={0,1}} - validation.cardinality("revision", getmetatable(self).path, cardinality, src) + local cardinality = {description={0,1}, reference={0,1}} + validation.cardinality("revision", getmetatable(self).path, cardinality, src) end -- Yang Module Module = {} function Module.new(base, name, src) - local ret = {body={}, name=name, modules={}, revisions={}, + local ret = {body={}, name=name, modules={}, revisions={}, features={}, groupings={}, containers={}} - local self = setmetatable(ret, {__index=Module, path=name}) + local self = setmetatable(ret, {__index=Module, path=name}) - -- TODO: remove me when proper loading support exists. - if not src then return self end + -- TODO: remove me when proper loading support exists. + if not src then return self end - -- Add self to path cache - base:add_cache(name, self) + -- Add self to path cache + base:add_cache(name, self) - -- Validate the statements first. - self:validate_schema(src) + -- Validate the statements first. + self:validate_schema(src) - -- Set the meta information about the module - self.namespace = src.namespace[1].argument - self.prefix = src.prefix[1].argument + -- Set the meta information about the module + self.namespace = src.namespace[1].argument + self.prefix = src.prefix[1].argument - if src.organization then - self.organization = src.organization[1].argument - end - if src.contact then - self.contact = src.contact[1].argument - end + if src.organization then + self.organization = src.organization[1].argument + end - if src.description then - self.description = src.description[1].argument - end + if src.contact then + self.contact = src.contact[1].argument + end + + if src.description then + self.description = src.description[1].argument + end - -- Now handle the imports, as other things may reference them. - if src.import then - for _, mod in pairs(src.import) do - self.modules[mod.argument] = Module.new(base, mod.argument) - - -- Ask the module to find and load itself. - self.modules[mod.argument]:load() - - -- TODO Add module to path cache. - end - end - - -- Handle revisions - if src.revision then - for _, r in pairs(src.revision) do - local path = ret.name.."."..r.argument - self.revisions[r.argument] = Revision.new(base, path, r.statements) - end - end - - -- Feature statements - if src.feature then - for _, f in pairs(src.feature) do - local path = ret.name.."."..f.argument - self.features[f.argument] = Feature.new(base, path, f.statements) - end - end - - -- Leaf statements - if src.leaf then - for _, leaf in pairs(src.leaf) do - end - end - - -- List statements - if src.grouping then - for _, g in pairs(src.grouping) do - local path = ret.name.."."..g.argument - self.groupings[g.argument] = Grouping.new(base, path, g.statements) - end - end - - -- Containers - if src.container then - for _, c in pairs(src.container) do - local path = ret.name.."."..c.argument - self.containers[c.argument] = Container.new(base, path, c.statements) - end - end - - return self + -- Now handle the imports, as other things may reference them. + if src.import then + for _, mod in pairs(src.import) do + self.modules[mod.argument] = Module.new(base, mod.argument) + + -- Ask the module to find and load itself. + self.modules[mod.argument]:load() + end + end + + -- Handle revisions + if src.revision then + for _, r in pairs(src.revision) do + local path = ret.name.."."..r.argument + self.revisions[r.argument] = Revision.new(base, path, r.statements) + end + end + + -- Feature statements + if src.feature then + for _, f in pairs(src.feature) do + local path = ret.name.."."..f.argument + self.features[f.argument] = Feature.new(base, path, f.statements) + end + end + + -- Leaf statements + if src.leaf then + for _, leaf in pairs(src.leaf) do + end + end + + -- List statements + if src.grouping then + for _, g in pairs(src.grouping) do + local path = ret.name.."."..g.argument + self.groupings[g.argument] = Grouping.new(base, path, g.statements) + end + end + + -- Containers + if src.container then + for _, c in pairs(src.container) do + local path = ret.name.."."..c.argument + self.containers[c.argument] = Container.new(base, path, c.statements) + end + end + return self end function Module:load() - -- TODO: Find the file and load it. + -- TODO: Find the file and load it. end function Module:validate_schema(src) - local cardinality = {contact={0,1}, description={0,1}, namespace={1,1}, - organization={0,1}, prefix={1,1}, reference={0,1}} - cardinality["yang-version"] = {0,1} - validation.cardinality("module", getmetatable(self).path, cardinality, src) + local cardinality = {contact={0,1}, description={0,1}, namespace={1,1}, + organization={0,1}, prefix={1,1}, reference={0,1}} + cardinality["yang-version"] = {0,1} + validation.cardinality("module", getmetatable(self).path, cardinality, src) end function selftest() - local test_schema = [[module ietf-softwire { - namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; - prefix "softwire"; - - import ietf-inet-types {prefix inet; } - import ietf-yang-types {prefix yang; } - - organization "Softwire Working Group"; - - contact - " - Qi Sun sunqi.ietf@gmail.com - Hao Wang wangh13@mails.tsinghua.edu.cn - Yong Cui yong@csnet1.cs.tsinghua.edu.cn - Ian Farrer ian.farrer@telekom.de - Mohamed Boucadair mohamed.boucadair@orange.com - Rajiv Asati rajiva@cisco.com - "; - - description - "This document defines a YANG data model for the configuration and - management of IPv4-in-IPv6 Softwire Border Routers and Customer - Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T - Softwire mechanisms. - - Copyright (c) 2014 IETF Trust and the persons identified - as authors of the code. All rights reserved. - This version of this YANG module is part of RFC XXX; see the RFC - itself for full legal notices."; - - - revision 2015-09-30 { - description - "Version-04: fix YANG syntax; Add flags to map-rule; Remove - the map-rule-type element. "; - reference "tbc"; - } - - revision 2015-04-07 { - description - "Version-03: Integrate lw4over6; Updata state nodes; Correct - grammar errors; Reuse groupings; Update descriptions. - Simplify the model."; - reference "tbc"; - } - - revision 2015-02-10 { - description - "Version-02: Add notifications."; - reference "tbc"; - } - - - revision 2015-02-06 { - description - "Version-01: Correct grammar errors; Reuse groupings; Update - descriptions."; - reference "tbc"; - } - - revision 2015-02-02 { - description - "Initial revision."; - reference "tbc"; - } - - feature lw4over6 { - description - "Lightweight 4over6 moves the Network Address and Port - Translation (NAPT) function from the centralized DS-Lite tunnel - concentrator to the tunnel client located in the Customer - Premises Equipment (CPE). This removes the requirement for a - Carrier Grade NAT function in the tunnel concentrator and - reduces the amount of centralized state that must be held to a - per-subscriber level. In order to delegate the NAPT function - and make IPv4 Address sharing possible, port-restricted IPv4 - addresses are allocated to the CPEs."; - reference - "I-D.ietf-softwire-lw4over6"; - } - - feature map-e { + local test_schema = [[module ietf-softwire { + namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; + prefix "softwire"; + + import ietf-inet-types {prefix inet; } + import ietf-yang-types {prefix yang; } + + organization "Softwire Working Group"; + + contact + " + Qi Sun sunqi.ietf@gmail.com + Hao Wang wangh13@mails.tsinghua.edu.cn + Yong Cui yong@csnet1.cs.tsinghua.edu.cn + Ian Farrer ian.farrer@telekom.de + Mohamed Boucadair mohamed.boucadair@orange.com + Rajiv Asati rajiva@cisco.com + "; + description - "MAP-E is a mechanism for transporting IPv4 packets across an - IPv6 network using IP encapsulation, and a generic mechanism - for mapping between IPv6 addresses and IPv4 addresses and - transport layer ports."; - reference - "I-D.ietf-softwire-map"; - } - - grouping port-set { - description - "Use the PSID algorithm to represent a range of transport layer - ports."; - leaf offset { - type uint8 { - range 0..16; + "This document defines a YANG data model for the configuration and + management of IPv4-in-IPv6 Softwire Border Routers and Customer + Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T + Softwire mechanisms. + + Copyright (c) 2014 IETF Trust and the persons identified + as authors of the code. All rights reserved. + This version of this YANG module is part of RFC XXX; see the RFC + itself for full legal notices."; + + revision 2015-09-30 { + description + "Version-04: fix YANG syntax; Add flags to map-rule; Remove + the map-rule-type element. "; + + reference "tbc"; } - mandatory true; - description - "The number of offset bits. In Lightweight 4over6, the defaul - value is 0 for assigning one contiguous port range. In MAP-E/T, - the default value is 6, which excludes system ports by default - and assigns distributed port ranges. If the this parameter is - larger than 0, the value of offset MUST be greater than 0."; - } - leaf psid { - type uint16; - mandatory true; - description - "Port Set Identifier (PSID) value, which identifies a set - of ports algorithmically."; - } - leaf psid-len { - type uint8 { - range 0..16; + + revision 2015-04-07 { + description + "Version-03: Integrate lw4over6; Updata state nodes; Correct + grammar errors; Reuse groupings; Update descriptions. + Simplify the model."; + + reference "tbc"; + } + + revision 2015-02-10 { + description "Version-02: Add notifications."; + reference "tbc"; + } + + revision 2015-02-06 { + description + "Version-01: Correct grammar errors; Reuse groupings; Update + descriptions."; + + reference "tbc"; + } + + revision 2015-02-02 { + description "Initial revision."; + reference "tbc"; + } + + feature lw4over6 { + description + "Lightweight 4over6 moves the Network Address and Port + Translation (NAPT) function from the centralized DS-Lite tunnel + concentrator to the tunnel client located in the Customer + Premises Equipment (CPE). This removes the requirement for a + Carrier Grade NAT function in the tunnel concentrator and + reduces the amount of centralized state that must be held to a + per-subscriber level. In order to delegate the NAPT function + and make IPv4 Address sharing possible, port-restricted IPv4 + addresses are allocated to the CPEs."; + + reference "I-D.ietf-softwire-lw4over6"; + } + + feature map-e { + description + "MAP-E is a mechanism for transporting IPv4 packets across an + IPv6 network using IP encapsulation, and a generic mechanism + for mapping between IPv6 addresses and IPv4 addresses and + transport layer ports."; + + reference "I-D.ietf-softwire-map"; + } + + grouping port-set { + description + "Use the PSID algorithm to represent a range of transport layer + ports."; + + leaf offset { + type uint8 { + range 0..16; + } + mandatory true; + description + "The number of offset bits. In Lightweight 4over6, the default + value is 0 for assigning one contiguous port range. In MAP-E/T, + the default value is 6, which excludes system ports by default + and assigns distributed port ranges. If the this parameter is + larger than 0, the value of offset MUST be greater than 0."; + } + + leaf psid { + type uint16; + mandatory true; + description + "Port Set Identifier (PSID) value, which identifies a set + of ports algorithmically."; + } + + leaf psid-len { + type uint8 { + range 0..16; + } + mandatory true; + description + "The length of PSID, representing the sharing ratio for an + IPv4 address."; + } + } + + container softwire-config { + description + "The configuration data for Softwire instances. And the shared + data describes the softwire data model which is common to all of + the different softwire mechanisms, such as description."; + + leaf description { + type string; + description "A textual description of Softwire."; + } } - mandatory true; - description - "The length of PSID, representing the sharing ratio for an - IPv4 address."; - } - } - - container softwire-config { - description - "The configuration data for Softwire instances. And the shared - data describes the softwire data model which is common to all of - the different softwire mechanisms, such as description."; - leaf description { - type string; - description - "A textual description of Softwire."; - } - } }]] -- Convert the schema using the already tested parser. diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 989721b248..87dea9d952 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -52,80 +52,82 @@ function Base:error(path, node, msg, ...) end function Base:load(src) - src = helpers.extract_nodes(src) - if not src.module then - error(("%s: Expected 'module'"):format(self.filename)) - end - local mod = schema.Module.new( - self, - src.module[1].argument, - src.module[1].statements - ) - - self.schema[src.module[1].argument] = mod - -- TODO: don't use rawget here. - local data_root = rawget(self.data, "root") - data_root[mod.name] = helpers.Container.new(self, mod.name) - return self.schema + src = helpers.extract_nodes(src) + if not src.module then + error(("%s: Expected 'module'"):format(self.filename)) + end + local mod = schema.Module.new( + self, + src.module[1].argument, + src.module[1].statements + ) + self.schema[src.module[1].argument] = mod + + -- TODO: don't use rawget here. + local data_root = rawget(self.data, "root") + data_root[mod.name] = helpers.Container.new(self, mod.name) + return self.schema end function Base:get_schema(path) - -- Handle case when path is empty. (Provide with self.data) - if path == "" then - return self.data - end - - -- Look in the cache and hopefully return the schema node for the path. - local cache = getmetatable(self).path_cache - return cache[path] + -- Handle case when path is empty. (Provide with self.data) + if path == "" then + return self.data + end + + -- Look in the cache and hopefully return the schema node for the path. + local cache = getmetatable(self).path_cache + return cache[path] end function Base:add_cache(path, node) - getmetatable(self).path_cache[path] = node + getmetatable(self).path_cache[path] = node end function Base:produce_data_tree(schema_node, data_node) - if not (schema_node and data_node) then - for _, mod in pairs(self.schema) do - schema_node = mod - end - if not schema_node then error("Module cannot be resolved") end - data_node = self.data[schema_node.name] - end - path = getmetatable(schema_node).path - - if schema_node.containers then - for name, container in pairs(schema_node.containers) do - local new_path = path.."."..name - local new_node = helpers.Container.new(self, new_path) - -- TODO: change me, we shouldn't be using rawget here! - local root = rawget(data_node, "root") - root[name] = new_node - schema_node = self:get_schema(new_path) - self:produce_data_tree(schema_node, new_node) - end - end - - if schema_node.leaves then - for name, leaf in pairs(schema_node.leaves) do - -- TODO remove the use of rawget here, it's not good. - local root = rawget(data_node, "root") - root[name] = leaf:provide_box() - end - end - - return self.data + if not (schema_node and data_node) then + for _, mod in pairs(self.schema) do + schema_node = mod + end + + if not schema_node then error("Module cannot be resolved") end + data_node = self.data[schema_node.name] + end + path = getmetatable(schema_node).path + + if schema_node.containers then + for name, container in pairs(schema_node.containers) do + local new_path = path.."."..name + local new_node = helpers.Container.new(self, new_path) + + -- TODO: change me, we shouldn't be using rawget here! + local root = rawget(data_node, "root") + root[name] = new_node + schema_node = self:get_schema(new_path) + self:produce_data_tree(schema_node, new_node) + end + end + + if schema_node.leaves then + for name, leaf in pairs(schema_node.leaves) do + -- TODO remove the use of rawget here, it's not good. + local root = rawget(data_node, "root") + root[name] = leaf:provide_box() + end + end + + return self.data end function Base:add(key, node) - self.data[key] = node + self.data[key] = node end function load_schema(schema, filename) - local parsed_yang = parser.parse_string(schema, "selftest") - local base = Base.new() - base:load(parsed_yang) - return base + local parsed_yang = parser.parse_string(schema, "selftest") + local base = Base.new() + base:load(parsed_yang) + return base end function load_schema_file(filename) @@ -135,156 +137,158 @@ function load_schema_file(filename) end function selftest() - local test_yang = [[module ietf-softwire { - namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; - prefix "softwire"; - - import ietf-inet-types {prefix inet; } - import ietf-yang-types {prefix yang; } - - organization "Softwire Working Group"; - - contact - " - Qi Sun sunqi.ietf@gmail.com - Hao Wang wangh13@mails.tsinghua.edu.cn - Yong Cui yong@csnet1.cs.tsinghua.edu.cn - Ian Farrer ian.farrer@telekom.de - Mohamed Boucadair mohamed.boucadair@orange.com - Rajiv Asati rajiva@cisco.com - "; - - description - "This document defines a YANG data model for the configuration and - management of IPv4-in-IPv6 Softwire Border Routers and Customer - Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T - Softwire mechanisms. - - Copyright (c) 2014 IETF Trust and the persons identified - as authors of the code. All rights reserved. - This version of this YANG module is part of RFC XXX; see the RFC - itself for full legal notices."; - - - revision 2015-09-30 { - description - "Version-04: fix YANG syntax; Add flags to map-rule; Remove - the map-rule-type element. "; - reference "tbc"; - } - - revision 2015-04-07 { - description - "Version-03: Integrate lw4over6; Updata state nodes; Correct - grammar errors; Reuse groupings; Update descriptions. - Simplify the model."; - reference "tbc"; - } - - revision 2015-02-10 { - description - "Version-02: Add notifications."; - reference "tbc"; - } - - - revision 2015-02-06 { - description - "Version-01: Correct grammar errors; Reuse groupings; Update - descriptions."; - reference "tbc"; - } - - revision 2015-02-02 { - description - "Initial revision."; - reference "tbc"; - } - - feature lw4over6 { - description - "Lightweight 4over6 moves the Network Address and Port - Translation (NAPT) function from the centralized DS-Lite tunnel - concentrator to the tunnel client located in the Customer - Premises Equipment (CPE). This removes the requirement for a - Carrier Grade NAT function in the tunnel concentrator and - reduces the amount of centralized state that must be held to a - per-subscriber level. In order to delegate the NAPT function - and make IPv4 Address sharing possible, port-restricted IPv4 - addresses are allocated to the CPEs."; - reference - "I-D.ietf-softwire-lw4over6"; - } + local test_yang = [[module ietf-softwire { + namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; + prefix "softwire"; + + import ietf-inet-types {prefix inet; } + import ietf-yang-types {prefix yang; } + + organization "Softwire Working Group"; + + contact + " + Qi Sun sunqi.ietf@gmail.com + Hao Wang wangh13@mails.tsinghua.edu.cn + Yong Cui yong@csnet1.cs.tsinghua.edu.cn + Ian Farrer ian.farrer@telekom.de + Mohamed Boucadair mohamed.boucadair@orange.com + Rajiv Asati rajiva@cisco.com + "; - feature map-e { description - "MAP-E is a mechanism for transporting IPv4 packets across an - IPv6 network using IP encapsulation, and a generic mechanism - for mapping between IPv6 addresses and IPv4 addresses and - transport layer ports."; - reference - "I-D.ietf-softwire-map"; - } + "This document defines a YANG data model for the configuration and + management of IPv4-in-IPv6 Softwire Border Routers and Customer + Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T + Softwire mechanisms. + + Copyright (c) 2014 IETF Trust and the persons identified + as authors of the code. All rights reserved. + This version of this YANG module is part of RFC XXX; see the RFC + itself for full legal notices."; + + + revision 2015-09-30 { + description + "Version-04: fix YANG syntax; Add flags to map-rule; Remove + the map-rule-type element. "; + + reference "tbc"; + } + + revision 2015-04-07 { + description + "Version-03: Integrate lw4over6; Updata state nodes; Correct + grammar errors; Reuse groupings; Update descriptions. + Simplify the model."; + + reference "tbc"; + } + + revision 2015-02-10 { + description "Version-02: Add notifications."; + reference "tbc"; + } + + revision 2015-02-06 { + description + "Version-01: Correct grammar errors; Reuse groupings; Update + descriptions."; + + reference "tbc"; + } + + revision 2015-02-02 { + description "Initial revision."; + reference "tbc"; + } - grouping port-set { - description - "Use the PSID algorithm to represent a range of transport layer - ports."; - leaf offset { - type uint8 { - range 0..16; + feature lw4over6 { + description + "Lightweight 4over6 moves the Network Address and Port + Translation (NAPT) function from the centralized DS-Lite tunnel + concentrator to the tunnel client located in the Customer + Premises Equipment (CPE). This removes the requirement for a + Carrier Grade NAT function in the tunnel concentrator and + reduces the amount of centralized state that must be held to a + per-subscriber level. In order to delegate the NAPT function + and make IPv4 Address sharing possible, port-restricted IPv4 + addresses are allocated to the CPEs."; + reference "I-D.ietf-softwire-lw4over6"; } - mandatory true; - description - "The number of offset bits. In Lightweight 4over6, the defaul - value is 0 for assigning one contiguous port range. In MAP-E/T, - the default value is 6, which excludes system ports by default - and assigns distributed port ranges. If the this parameter is - larger than 0, the value of offset MUST be greater than 0."; + + feature map-e { + description + "MAP-E is a mechanism for transporting IPv4 packets across an + IPv6 network using IP encapsulation, and a generic mechanism + for mapping between IPv6 addresses and IPv4 addresses and + transport layer ports."; + + reference "I-D.ietf-softwire-map"; } - leaf psid { - type uint16; - mandatory true; + + grouping port-set { description - "Port Set Identifier (PSID) value, which identifies a set - of ports algorithmically."; - } - leaf psid-len { - type uint8 { - range 0..16; + "Use the PSID algorithm to represent a range of transport layer + ports."; + + leaf offset { + type uint8 { + range 0..16; + } + mandatory true; + description + "The number of offset bits. In Lightweight 4over6, the defaul + value is 0 for assigning one contiguous port range. In MAP-E/T, + the default value is 6, which excludes system ports by default + and assigns distributed port ranges. If the this parameter is + larger than 0, the value of offset MUST be greater than 0."; } - mandatory true; - description - "The length of PSID, representing the sharing ratio for an - IPv4 address."; - } - } - - container softwire-config { - description - "The configuration data for Softwire instances. And the shared - data describes the softwire data model which is common to all of - the different softwire mechanisms, such as description."; - leaf description { - type string; + + leaf psid { + type uint16; + mandatory true; + description + "Port Set Identifier (PSID) value, which identifies a set + of ports algorithmically."; + } + + leaf psid-len { + type uint8 { + range 0..16; + } + mandatory true; + description + "The length of PSID, representing the sharing ratio for an + IPv4 address."; + } + } + + container softwire-config { description - "A textual description of Softwire."; - } + "The configuration data for Softwire instances. And the shared + data describes the softwire data model which is common to all of + the different softwire mechanisms, such as description."; - leaf testvalue { - type uint8; - description "Test value for unsigned 8 bit integers."; - } - } + leaf description { + type string; + description "A textual description of Softwire."; + } + + leaf testvalue { + type uint8; + description "Test value for unsigned 8 bit integers."; + } + } }]] - local base = load_schema(test_yang) - local data = base:produce_data_tree() + local base = load_schema(test_yang) + local data = base:produce_data_tree() - -- Set the description leaf - data["ietf-softwire"]["softwire-config"].description = "I am a description" - assert(data["ietf-softwire"]["softwire-config"].description == "I am a description") + -- Set the description leaf + data["ietf-softwire"]["softwire-config"].description = "I am a description" + assert(data["ietf-softwire"]["softwire-config"].description == "I am a description") - -- Set the testvalue - data["ietf-softwire"]["softwire-config"].testvalue = 72 - assert(data["ietf-softwire"]["softwire-config"].testvalue == 72) + -- Set the testvalue + data["ietf-softwire"]["softwire-config"].testvalue = 72 + assert(data["ietf-softwire"]["softwire-config"].testvalue == 72) end \ No newline at end of file From 2e107e63b9546efb774aca0b7de9579936cb06bc Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 10 May 2016 18:10:01 +0200 Subject: [PATCH 015/631] Switch to use syscall.helpers and remove unused meta I was under the impression syscall.helpers didn't work for multi-character delimiters however after testing it turns out I was mistaken, I've switched back to using the syscall version. I also removed a unused variable that was brought to my attaintion. I have also changed to using the ffi's typeof structs which should speed things up a bit. --- src/lib/yang/helpers.lua | 30 ---------------------- src/lib/yang/schema.lua | 51 ++++++++++++++++--------------------- src/lib/yang/validation.lua | 18 ++++++------- src/lib/yang/yang.lua | 15 ++++++----- 4 files changed, 39 insertions(+), 75 deletions(-) diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index 7b8d686c28..31c6e7c578 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -1,25 +1,6 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -function split(delimiter, text) - -- This is taken from syslog. - if delimiter == "" then return {text} end - if #text == 0 then return {} end - local list = {} - local pos = 1 - while true do - local first, last = text:find(delimiter, pos) - if first then - list[#list + 1] = text:sub(pos, first - 1) - pos = last + 1 - else - list[#list + 1] = text:sub(pos) - break - end - end - return list -end - function extract_nodes(schema) local nodes = {} for _, v in pairs(schema) do @@ -39,7 +20,6 @@ end Container = {} function Container.new(base, path) local ret = {root={}, base=base, path=path} - local meta = {} return setmetatable(ret, { __newindex = function (t, k, v) -- Validate the value prior to setting it. @@ -66,14 +46,4 @@ function Container.new(base, path) end end }) -end - -function selftest() - local result = split("%.%.", "0..9") - assert(result[1] == "0") - assert(result[2] == "9") - - local result = split("%-", "hello-lua") - assert(result[1] == "hello") - assert(result[2] == "lua") end \ No newline at end of file diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index e8aecd0cb4..5a09b3b8cd 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -12,27 +12,26 @@ module(..., package.seeall) local validation = require("lib.yang.validation") local helpers = require("lib.yang.helpers") +local h = require("syscall.helpers") -- Use ffi types because they will validate that numeric values are being -- provided. The downside is that integer overflow could occur on these. This -- route has been selected as validation will be faster than attempting to -- validate in Lua. local ffi = require("ffi") -ffi.cdef[[ -typedef struct { int8_t value; } int8box; -typedef struct { int16_t value; } int16box; -typedef struct { int32_t value; } int32box; -typedef struct { int64_t value; } int64box; -typedef struct { uint8_t value; } uint8box; -typedef struct { uint16_t value; } uint16box; -typedef struct { uint32_t value; } uint32box; -typedef struct { uint64_t value; } uint64box; -typedef struct { double value; } decimal64box; -]] +local int8box = ffi.typeof("struct { int8_t value; }") +local int16box = ffi.typeof("struct { int16_t value; }") +local int32box = ffi.typeof("struct { int32_t value; }") +local int64box = ffi.typeof("struct { int64_t value; }") +local uint8box = ffi.typeof("struct { uint8_t value; }") +local uint16box = ffi.typeof("struct { uint16_t value; }") +local uint32box = ffi.typeof("struct { uint32_t value; }") +local uint64box = ffi.typeof("struct { uint64_t value; }") +local decimal64box = ffi.typeof("struct { double value; }") local Leaf = {} function Leaf.new(base, path, src) - self = setmetatable({}, {__index=Leaf, path=path}) + local self = setmetatable({}, {__index=Leaf, path=path}) -- Parse the schema to find the metadata self:validate_schema(src) @@ -42,7 +41,7 @@ function Leaf.new(base, path, src) if src.type[1].statements then local typeinfo = src.type[1].statements if typeinfo.range then - local range = helpers.split("%.%.", typeinfo.range[1].argument) + local range = h.split("%.%.", typeinfo.range[1].argument) self.range = {tonumber(range[1]), tonumber(range[2])} end end @@ -101,23 +100,23 @@ function Leaf:provide_box() local box if self.type == "int8" then - box = ffi.new("int8box") + box = int8box() elseif self.type == "int16" then - box = ffi.new("int16box") + box = int16box() elseif self.type == "int32" then - box = ffi.new("int32box") + box = int32box() elseif self.type == "int64" then - box = ffi.new("int64box") + box = int64box() elseif self.type == "uint8" then - box = ffi.new("uint8box") + box = uint8box() elseif self.type == "uint16" then - box = ffi.new("uint16box") + box = uint16box() elseif self.type == "uint32" then - box = ffi.new("uint32box") + box = uint32box() elseif self.type == "uint64" then - box = ffi.new("uint64box") + box = uint64box() elseif self.type == "decimal64" then - box = ffi.new("decimal64box") + box = decimal64box() elseif self.type == "string" then box = {} elseif self.type == "boolean" then @@ -262,7 +261,7 @@ function Revision.new(base, path, src) self.description = src.description[1].argument end - if src.description then + if src.reference then self.reference = src.reference[1].argument end return self @@ -331,12 +330,6 @@ function Module.new(base, name, src) end end - -- Leaf statements - if src.leaf then - for _, leaf in pairs(src.leaf) do - end - end - -- List statements if src.grouping then for _, g in pairs(src.grouping) do diff --git a/src/lib/yang/validation.lua b/src/lib/yang/validation.lua index 84847ee41d..a699fc5269 100644 --- a/src/lib/yang/validation.lua +++ b/src/lib/yang/validation.lua @@ -2,15 +2,15 @@ module(..., package.seeall) function cardinality(kw, path, statements, haystack) for s, c in pairs(statements) do - if (c[1] >= 1 and (not haystack[s])) or (#statements[s] < c[1] and #statements[s] > c[2]) then - if c[1] == c[2] then - error(("Expected %d %s statement(s) in %s:%s"):format( - c[1], s, kw, path)) - else - error( - ("Expected between %d and %d of %s statement(s) in %s:%s"):format( - c[1], c[2], s, kw, path)) - end + if (c[1] >= 1 and (not haystack[s])) or + (#statements[s] < c[1] and #statements[s] > c[2]) then + if c[1] == c[2] then + error(("Expected %d %s statement(s) in %s:%s"):format( + c[1], s, kw, path)) + else + local err = "Expected between %d and %d of %s statement(s) in %s:%s" + error((err):format(c[1], c[2], s, kw, path)) + end end end return true diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 87dea9d952..07eee8ba60 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -5,11 +5,11 @@ -- To use this you should use the helper functions: -- - load_schema -- - load_schema_file --- The former takes in a yang string and builds a Base table, similiaraly the +-- The former takes in a yang string and builds a Base table, similarly the -- load_schema_file takes a schema file and builds a Base table from that. -- -- Once you have the base file you can inspect the schema, data and set values. --- Examaple: +-- Example: -- local base = load_schema[[ -- module example { -- namespace "exampleinc:example"; @@ -38,12 +38,13 @@ module(..., package.seeall) local schema = require("lib.yang.schema") local helpers = require("lib.yang.helpers") local parser = require("lib.yang.parser") +local Container = helpers.Container local Base = {} function Base.new(filename) - ret = {schema={}, filename=filename} - self = setmetatable(ret, {__index=Base, path_cache={}}) - self.data = helpers.Container.new(self, "") + local ret = {schema={}, filename=filename} + local self = setmetatable(ret, {__index=Base, path_cache={}}) + self.data = Container.new(self, "") return self end @@ -65,7 +66,7 @@ function Base:load(src) -- TODO: don't use rawget here. local data_root = rawget(self.data, "root") - data_root[mod.name] = helpers.Container.new(self, mod.name) + data_root[mod.name] = Container.new(self, mod.name) return self.schema end @@ -98,7 +99,7 @@ function Base:produce_data_tree(schema_node, data_node) if schema_node.containers then for name, container in pairs(schema_node.containers) do local new_path = path.."."..name - local new_node = helpers.Container.new(self, new_path) + local new_node = Container.new(self, new_path) -- TODO: change me, we shouldn't be using rawget here! local root = rawget(data_node, "root") From f3d5e36ec55e0ee044fe77d915a6e64695795a77 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 11 May 2016 14:40:14 +0200 Subject: [PATCH 016/631] Add the missing file descriptor closing --- src/lib/yang/yang.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 07eee8ba60..2551d8d578 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -134,6 +134,7 @@ end function load_schema_file(filename) local file_in = assert(io.open(filename)) local contents = file_in:read("*a") + file_in:close() return load_schema(contents, filename) end From cd1fd3db1ee99bc59fad7f2bd452353372aa2b2e Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 19 May 2016 17:13:59 +0200 Subject: [PATCH 017/631] Add IPv4 and IPv6 box types as well as fixing bugs --- src/lib/yang/helpers.lua | 148 ++++++++++++++++++++++++++++++++++++++- src/lib/yang/schema.lua | 74 +++++++++++++++----- src/lib/yang/yang.lua | 111 +++++++++++++++++++++++++---- 3 files changed, 298 insertions(+), 35 deletions(-) diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index 31c6e7c578..e102b0bba7 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -1,6 +1,9 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) +local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") + function extract_nodes(schema) local nodes = {} for _, v in pairs(schema) do @@ -17,21 +20,92 @@ function extract_nodes(schema) return nodes end +IPv4Box = {} +function IPv4Box.new(address) + local ret = {root={}} + if address then ret.value = ipv4:pton(address) end + return setmetatable(ret, { + __index = function (t, k) return rawget(ret, "root").value end, + __newindex = function(t, k, v) + rawget(ret, "root").value = ipv4:pton(v) + end + }) +end + +IPv6Box = {} +function IPv6Box.new(address) + local ret = {root={}} + if address then ret.value = ipv6:pton(address) end + return setmetatable(ret, { + __index = function (t, k) return rawget(ret, "root").value end, + __newindex = function(t, k, v) + rawget(ret, "root").value = ipv6:pton(v) + end + }) +end + +Enum = {} +function Enum.new(options) + local ret = {root={}} + for i, option in pairs(options) do + ret[i] = option + end + return setmetatable(ret, { + __index = function (t, k) return rawget(ret, "root")[k].value end, + __newindex = function(t, k, v) + if rawget(ret, v) then rawget(ret, "root")[k].value = v end + end + }) +end + +Union = {} +function Union.new(types) + local ret = {root={}, types={}} + + -- Importing here to prevent cyclic imports. + local Leaf = require("lib.yang.schema").Leaf + for _, name in pairs(types) do + ret.types[name] = Leaf.provide_box(nil, name) + end + + return setmetatable(ret, { + __index = function (t, k) + return rawget(ret, "root").box.value + end, + __newindex = function(t, k, v) + local function setbox(dest, v) dest.value = v end + local root = rawget(ret, "root") + for name, box in pairs(rawget(ret, "types")) do + local valid = pcall(setbox, box, v) + if valid then + root.box = box + return -- don't want to continue. + end + end + error(("Unable to find matching type for '%s' (%s)"):format(v, type(v))) + end + }) +end + Container = {} function Container.new(base, path) local ret = {root={}, base=base, path=path} return setmetatable(ret, { __newindex = function (t, k, v) -- Validate the value prior to setting it. - local schema = ret.base:get_schema(ret.path.."."..k) + local node_path = ret.path.."."..k + local schema = assert( + ret.base:get_schema(ret.path.."."..k), + ("No schema found at: %s.%s"):format(ret.path, k) + ) + if schema.validation then for _, validation in pairs(schema.validation) do validation(v) end end - local box = ret.root[k] - if box == nil then error("Unknown leaf '"..k.."'") end + local box = assert(ret.root[k], "Unknown leaf '"..node_path.."'") box.value = v end, __index = function(t, k) @@ -46,4 +120,72 @@ function Container.new(base, path) end end }) +end + +function pp(x, n) + if n ~= nil and n <= 0 then return nil end + if type(x) == "table" then + io.write("{") + local first = true + for k,v in pairs(x) do + if not first then + io.write(", ") + end + io.write(k.."=") + if n then pp(v, n-1) else pp(v) end + first = false + end + io.write("}") + elseif type(x) == "string" then + io.write(x) + else + io.write((""):format(type(x))) + end +end + +-- Functions to help testing. +function asserterror(func, ...) + local success, val = pcall(func, ...) + if success then + error(("Asserterror failed! Returned with '%s'"):format(val)) + end +end + +function setvalue(t, k, v) + t[k] = v +end + +function selftest() + local base = require("lib.yang.yang").Base.new() + -- Register a fake schema to satisfy the requirements. + base:add_cache("mod.testbox", {}) + + local con = Container.new(base, "mod") + local root = rawget(con, "root") + + -- Test the union box + root.testbox = Union.new({"uint8", "inet:ipv4-address"}) + + -- Allow a uint8 value. + con.testbox = 72 + assert(con.testbox == 72) + + -- Allow a valid IPv4 address + con.testbox = "8.8.8.8" + assert(ipv4:ntop(con.testbox) == "8.8.8.8") + + -- setting "should fail" is not uint8 or valid IPv4 address so should fail. + asserterror(setvalue, con, "testbox", "should fail") + + + -- Test the IPv4 box. + root.testbox = IPv4Box.new() + + -- Set it to a valid IPv4 address + con.testbox = "8.8.8.8" + assert(ipv4:ntop(con.testbox) == "8.8.8.8") + + -- Set it to something erroneous and check it fails + con.testbox = "256.8.8.8" + assert(ipv4:ntop(con.testbox) == "256.8.8.8") end \ No newline at end of file diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 5a09b3b8cd..7327578af5 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -29,7 +29,7 @@ local uint32box = ffi.typeof("struct { uint32_t value; }") local uint64box = ffi.typeof("struct { uint64_t value; }") local decimal64box = ffi.typeof("struct { double value; }") -local Leaf = {} +Leaf = {} function Leaf.new(base, path, src) local self = setmetatable({}, {__index=Leaf, path=path}) @@ -43,6 +43,11 @@ function Leaf.new(base, path, src) if typeinfo.range then local range = h.split("%.%.", typeinfo.range[1].argument) self.range = {tonumber(range[1]), tonumber(range[2])} + elseif typeinfo.enum then + self.enums = {} + for _, v in pairs(typeinfo.enum) do + self.enums[v.argument] = v.argument + end end end if src.description then @@ -81,6 +86,14 @@ function Leaf.new(base, path, src) end end end + if self.enums then + if not self.validation then self.validation = {} end + self.validation[#self.validation + 1] = function (v) + if v and not self.enums[v] then + self:error("Value '%s' is not one of the Enum values", v) + end + end + end return self end @@ -96,34 +109,44 @@ function Leaf:validate_schema(schema) validation.cardinality("leaf", getmetatable(self).path, cardinality, schema) end -function Leaf:provide_box() +function Leaf:provide_box(leaf_type, statements) local box - if self.type == "int8" then + if not leaf_type then leaf_type = self.type end + + if leaf_type == "int8" then box = int8box() - elseif self.type == "int16" then + elseif leaf_type == "int16" then box = int16box() - elseif self.type == "int32" then + elseif leaf_type == "int32" then box = int32box() - elseif self.type == "int64" then + elseif leaf_type == "int64" then box = int64box() - elseif self.type == "uint8" then + elseif leaf_type == "uint8" then box = uint8box() - elseif self.type == "uint16" then + elseif leaf_type == "uint16" then box = uint16box() - elseif self.type == "uint32" then + elseif leaf_type == "uint32" then box = uint32box() - elseif self.type == "uint64" then + elseif leaf_type == "uint64" then box = uint64box() - elseif self.type == "decimal64" then + elseif leaf_type == "decimal64" then box = decimal64box() - elseif self.type == "string" then + elseif leaf_type == "string" then box = {} - elseif self.type == "boolean" then + elseif leaf_type == "boolean" then box = {} + elseif leaf_type == "enumeration" then + box = helpers.Enum.new(self.enums) + elseif leaf_type == "union" then + box = helpers.Union.new(statements) + elseif leaf_type == "inet:ipv4-address" then + box = helpers.IPv4Box.new() + elseif leaf_type == "inet:ipv6-address" then + box = helpers.IPv6Box.new() else - error(("Unknown type '%s' for leaf '%s'"):format( - leaf_type, self.config.name)) + local path = self and getmetatable(self).path or "" + error(("Unknown type '%s' for leaf"):format(path, leaf_type)) end return box @@ -159,7 +182,7 @@ end -- Yang list local List = {} -function List.new(path, src) +function List.new(base, path, src) local self = setmetatable({}, {__index=List, path=path}) self:validate_schema(src) @@ -222,7 +245,7 @@ end local Container = {} function Container.new(base, path, src) - local ret = {leaves={}} + local ret = {leaves={}, containers={}} local self = setmetatable(ret, {__index=Container, path=path}) self:validate_schema(src) @@ -240,6 +263,22 @@ function Container.new(base, path, src) end end + -- Include other containers + if src.container then + for _, container in pairs(src.container) do + local container_path = path.."."..container.argument + self.containers[container.argument] = Container.new( + base, + container_path, + container.statements + ) + end + end + + if src.uses then + self.uses = src.uses[1].argument + end + return self end @@ -514,7 +553,6 @@ function selftest() local mod = Module.new(base, schema.module[1].argument, schema.module[1].statements) - print("module: ", mod.name) assert(mod.name == "ietf-softwire") assert(mod.namespace == "urn:ietf:params:xml:ns:yang:ietf-softwire") assert(mod.prefix == "softwire") diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 2551d8d578..25f563bb5a 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -40,7 +40,7 @@ local helpers = require("lib.yang.helpers") local parser = require("lib.yang.parser") local Container = helpers.Container -local Base = {} +Base = {} function Base.new(filename) local ret = {schema={}, filename=filename} local self = setmetatable(ret, {__index=Base, path_cache={}}) @@ -85,11 +85,22 @@ function Base:add_cache(path, node) getmetatable(self).path_cache[path] = node end +function Base:get_module() + local schema_mod + for _, mod in pairs(self.schema) do + schema_mod = mod + end + + if not schema_mod then + error("Module cannot be resolved") + end + + return schema_mod +end + function Base:produce_data_tree(schema_node, data_node) if not (schema_node and data_node) then - for _, mod in pairs(self.schema) do - schema_node = mod - end + schema_node = self:get_module() if not schema_node then error("Module cannot be resolved") end data_node = self.data[schema_node.name] @@ -105,7 +116,26 @@ function Base:produce_data_tree(schema_node, data_node) local root = rawget(data_node, "root") root[name] = new_node schema_node = self:get_schema(new_path) + data_node = new_node self:produce_data_tree(schema_node, new_node) + + -- If the container has a "uses" statement we must copy across the + -- leaves from the container it references to this container. + if container.uses then + local root = rawget(data_node, "root") + local grouping = self:get_module().groupings[container.uses] + if not grouping then + self:error(path, name, "Cannot find grouping '%s'.", container.uses) + end + + -- Copy. + for name, leaf in pairs(grouping.leaves) do + -- We also need to register the schema node at the new path + local grouping_path = new_path.."."..name + self:add_cache(grouping_path, leaf) + root[name] = leaf:provide_box() + end + end end end @@ -116,7 +146,6 @@ function Base:produce_data_tree(schema_node, data_node) root[name] = leaf:provide_box() end end - return self.data end @@ -266,6 +295,62 @@ function selftest() } } + grouping binding-entry { + description + "The lwAFTR maintains an address binding table that contains + thebinding between the lwB4's IPv6 address, the allocated IPv4 + address and restricted port-set."; + + leaf binding-ipv6info { + type union { + type inet:ipv6-address; + type inet:ipv6-prefix; + } + + mandatory true; + description + "The IPv6 information for a binding entry. + If it's an IPv6 prefix, it indicates that + the IPv6 source address of the lwB4 is constructed + according to the description in RFC7596; + if it's an IPv6 address, it means the lwB4 uses"; + } + + leaf binding-ipv4-addr { + type inet:ipv4-address; + mandatory true; + description + "The IPv4 address assigned to the lwB4, which is + used as the IPv4 external address + for lwB4 local NAPT44."; + } + + container port-set { + description + "For Lightweight 4over6, the default value + of offset should be 0, to configure one contiguous + port range."; + uses port-set { + refine offset { + default "0"; + } + } + } + + leaf lwaftr-ipv6-addr { + type inet:ipv6-address; + mandatory true; + description "The IPv6 address for lwaftr."; + } + + leaf lifetime { + type uint32; + units seconds; + description "The lifetime for the binding entry"; + } + } + + container softwire-config { description "The configuration data for Softwire instances. And the shared @@ -277,20 +362,18 @@ function selftest() description "A textual description of Softwire."; } - leaf testvalue { - type uint8; - description "Test value for unsigned 8 bit integers."; + container testgroup { + uses binding-entry; } } }]] local base = load_schema(test_yang) local data = base:produce_data_tree() - -- Set the description leaf - data["ietf-softwire"]["softwire-config"].description = "I am a description" - assert(data["ietf-softwire"]["softwire-config"].description == "I am a description") + data["ietf-softwire"]["softwire-config"].description = "hello!" + assert(data["ietf-softwire"]["softwire-config"].description == "hello!") - -- Set the testvalue - data["ietf-softwire"]["softwire-config"].testvalue = 72 - assert(data["ietf-softwire"]["softwire-config"].testvalue == 72) + -- Test grouping. + data["ietf-softwire"]["softwire-config"].testgroup["lifetime"] = 72 + assert(data["ietf-softwire"]["softwire-config"].testgroup["lifetime"] == 72) end \ No newline at end of file From 4714f6b985d3e5ee98556a986696569f4c9396e5 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 13 Jun 2016 11:07:15 +0200 Subject: [PATCH 018/631] Add more statements, fix bugs and add more types --- src/lib/yang/helpers.lua | 227 +++++++++++++++++++++------ src/lib/yang/parser.lua | 2 +- src/lib/yang/schema.lua | 263 ++++++++++++++----------------- src/lib/yang/validation.lua | 17 -- src/lib/yang/yang.lua | 302 +++++++++++++----------------------- 5 files changed, 399 insertions(+), 412 deletions(-) delete mode 100644 src/lib/yang/validation.lua diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index e102b0bba7..5011d0dd06 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -1,13 +1,15 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) +local h = require("syscall.helpers") local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") +local corelib = require("core.lib") function extract_nodes(schema) local nodes = {} for _, v in pairs(schema) do - -- Recursively apply this. + -- Recursively apply this if v.statements then v.statements = extract_nodes(v.statements) end -- Add to the nodes table. @@ -21,39 +23,81 @@ function extract_nodes(schema) end IPv4Box = {} -function IPv4Box.new(address) +function IPv4Box.new() local ret = {root={}} - if address then ret.value = ipv4:pton(address) end return setmetatable(ret, { __index = function (t, k) return rawget(ret, "root").value end, __newindex = function(t, k, v) - rawget(ret, "root").value = ipv4:pton(v) + local ipv4boxed, err = ipv4:pton(v) + if ipv4boxed == false then error(err) end + rawget(ret, "root").value = ipv4boxed end }) end IPv6Box = {} -function IPv6Box.new(address) +function IPv6Box.new() local ret = {root={}} - if address then ret.value = ipv6:pton(address) end return setmetatable(ret, { __index = function (t, k) return rawget(ret, "root").value end, __newindex = function(t, k, v) - rawget(ret, "root").value = ipv6:pton(v) + local ipv6boxed, err = ipv6:pton(v) + if ipv6boxed == false then error(err) end + rawget(ret, "root").value = ipv6boxed end }) end +IPv4PrefixBox = {} +function IPv4PrefixBox.new() + local ret = {root={}} + return setmetatable(ret, { + __newindex = function (t, k, v) + -- split the netmask and prefix up. + local raw = h.split("/", v) + local addr, err = ipv4:pton(raw[1]) + if addr == false then error(err) end + local prefix = tonumber(raw[2]) + if prefix > 32 or prefix < 0 then + error(("The prefix length is invalid. (%s)"):format(raw[2])) + end + rawget(ret, "root").value = {addr, prefix} + end, + __index = function (t, k, v) return rawget(ret, "root").value end + }) +end + +IPv6PrefixBox = {} +function IPv6PrefixBox.new() + local ret = {root={}} + return setmetatable(ret, { + __newindex = function (t, k, v) + -- split the netmask and prefix up. + local raw = h.split("/", v) + local addr, err = ipv6:pton(raw[1]) + if addr == false then error(err) end + local prefix = tonumber(raw[2]) + if prefix > 128 or prefix < 0 then + error(("The prefix length is invalid. (%s)"):format(raw[2])) + end + rawget(ret, "root").value = {addr, prefix} + end, + __index = function (t, k, v) return rawget(ret, "root").value end + }) +end + Enum = {} function Enum.new(options) local ret = {root={}} - for i, option in pairs(options) do - ret[i] = option + local opts = {} + for _, option in pairs(options) do + opts[option] = option end return setmetatable(ret, { - __index = function (t, k) return rawget(ret, "root")[k].value end, + __index = function (t, k) return rawget(ret, "root").value end, __newindex = function(t, k, v) - if rawget(ret, v) then rawget(ret, "root")[k].value = v end + if opts[v] then rawget(ret, "root").value = v + else error("Value "..v.." is not a valid option for this Enum.") end end }) end @@ -65,7 +109,11 @@ function Union.new(types) -- Importing here to prevent cyclic imports. local Leaf = require("lib.yang.schema").Leaf for _, name in pairs(types) do - ret.types[name] = Leaf.provide_box(nil, name) + -- 9.12 specifies unions cannot contain "leafref" or "empty" + if name == "empty" or name == "leafref" then + error("Union type cannot contain 'empty' or 'leafref'") + end + ret.types[name] = Leaf:provide_box(name) end return setmetatable(ret, { @@ -111,36 +159,83 @@ function Container.new(base, path) __index = function(t, k) local table = rawget(t, "root") local prop = table[k] + if not prop then - return prop + -- This could be because it's a method defined on Container. + return Container[k] elseif prop.value == nil then return prop else return prop.value end - end + end, }) end -function pp(x, n) - if n ~= nil and n <= 0 then return nil end - if type(x) == "table" then - io.write("{") - local first = true - for k,v in pairs(x) do - if not first then - io.write(", ") - end - io.write(k.."=") - if n then pp(v, n-1) else pp(v) end - first = false - end - io.write("}") - elseif type(x) == "string" then - io.write(x) +function Container:set_template(template) + rawset(self, "template", template) +end +function Container:get_template(template) + return rawget(self, "template") +end + +function Container:add_item(item) + local root = rawget(self, "root") + local base = rawget(self, "base") + local path = rawget(self, "path") + + -- Verify that the item is being added to a list type. + local schema = base:get_schema(path) + if schema:get_type() ~= "list" then + error("Can't add item to '"..schema:get_type().."'") + end + + -- Each entry will have their own container which is a copy of the template. + local template = assert(self:get_template(), "Must set template to add item") + local con = template:duplicate() + + -- Find the leaves from the schema, if it's a group then find the parent. + local leaves + if schema.uses then + leaves = base:schema_for_uses(schema).leaves else - io.write((""):format(type(x))) + leaves = schema.leaves end + + -- Create a new table of item + leavesf + local data = corelib.deepcopy(item) + for name, leaf in pairs(leaves) do data[name] = leaf end + + -- Add the data to the container. + for name, leaf in pairs(data) do + if leaves[name] == nil then + error("Unknown field in list: '"..name.."'") + elseif item[name] == nil and leaves[name].mandatory then + error("Field "..name.." not provided but is mandatory.") + elseif item[name] then + con[name] = item[name] + end + end + + -- Add the container entry to container. + table.insert(root, con) +end + +function Container:add_container(name) + self:add_to_root(name, Container.new(self, name)) +end + +function Container:add_to_root(key, value) + local root = rawget(self, "root") + root[key] = value +end + +function Container:duplicate() + -- Produces and returns a duplicate of the container + local root = corelib.deepcopy(rawget(self, "root")) + local copy = Container.new(rawget(self, "base"), rawget(self, "path")) + rawset(copy, "root", root) + return copy end -- Functions to help testing. @@ -151,6 +246,22 @@ function asserterror(func, ...) end end +function cardinality(kw, path, statements, haystack) + for s, c in pairs(statements) do + if (c[1] >= 1 and (not haystack[s])) or + (#statements[s] < c[1] and #statements[s] > c[2]) then + if c[1] == c[2] then + error(("Expected %d %s statement(s) in %s:%s"):format( + c[1], s, kw, path)) + else + local err = "Expected between %d and %d of %s statement(s) in %s:%s" + error((err):format(c[1], c[2], s, kw, path)) + end + end + end + return true +end + function setvalue(t, k, v) t[k] = v end @@ -163,29 +274,53 @@ function selftest() local con = Container.new(base, "mod") local root = rawget(con, "root") - -- Test the union box + -- Test the union box (both uint8, ipv4 - valid and invalid data) root.testbox = Union.new({"uint8", "inet:ipv4-address"}) - - -- Allow a uint8 value. con.testbox = 72 assert(con.testbox == 72) - - -- Allow a valid IPv4 address con.testbox = "8.8.8.8" assert(ipv4:ntop(con.testbox) == "8.8.8.8") - - -- setting "should fail" is not uint8 or valid IPv4 address so should fail. asserterror(setvalue, con, "testbox", "should fail") - - -- Test the IPv4 box. + -- Test the IPv4 box (both valid and invalid data). root.testbox = IPv4Box.new() - - -- Set it to a valid IPv4 address con.testbox = "8.8.8.8" assert(ipv4:ntop(con.testbox) == "8.8.8.8") + asserterror(setvalue, con, "testbox", "256.8.8.8") + + -- Test the IPv6 box (both valid and invalid data). + root.testbox = IPv6Box.new() + con.testbox = "::1" + assert(ipv6:ntop(con.testbox) == "::1") + asserterror(setvalue, con, "testbox", "not ipv6") - -- Set it to something erroneous and check it fails - con.testbox = "256.8.8.8" - assert(ipv4:ntop(con.testbox) == "256.8.8.8") -end \ No newline at end of file + -- Testing enums (both valid and invalid values) + root.testbox = Enum.new({"Banana", "Apple"}) + con.testbox = "Banana" + assert(con.testbox == "Banana") + con.testbox = "Apple" + assert(con.testbox == "Apple") + asserterror(setvalue, con, "testbox", "Pear") + + -- Testing ipv4 prefix + root.testbox = IPv4PrefixBox.new() + con.testbox = "192.168.0.0/24" + local rtn = assert(con.testbox) + assert(ipv4:ntop(rtn[1]) == "192.168.0.0") + assert(rtn[2] == 24) + asserterror(setvalue, con, "testbox", "Incorrect value") + asserterror(setvalue, con, "testbox", "192.168.0.0/33") + asserterror(setvalue, con, "testbox", "192.168.0.0/-1") + asserterror(setvalue, con, "testbox", "192.256.0.0/24") + + -- Testing ipv6 prefix + root.testbox = IPv6PrefixBox.new() + con.testbox = "2001:db8::/32" + local rtn = assert(con.testbox) + assert(ipv6:ntop(rtn[1]) == "2001:db8::") + assert(rtn[2] == 32) + asserterror(setvalue, con, "testbox", "Incorrect value") + asserterror(setvalue, con, "testbox", "2001:db8::/129") + asserterror(setvalue, con, "testbox", "2001:db8::/-1") + asserterror(setvalue, con, "testbox", "FFFFF:db8::/32") +end diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index a8d4c5074f..f4031fd9f7 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -356,5 +356,5 @@ function selftest() argument="port", statements={{keyword="type"}}}}) -- Expects tests to be run from the "src" directory at the root of the repo - parse_file("lib/yang/example.yang") + parse_file("src/lib/yang/example.yang") end diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 7327578af5..2d82f33469 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -10,7 +10,6 @@ -- This relies on the "Base" table which can be found in the yang.lua file. module(..., package.seeall) -local validation = require("lib.yang.validation") local helpers = require("lib.yang.helpers") local h = require("syscall.helpers") @@ -28,6 +27,7 @@ local uint16box = ffi.typeof("struct { uint16_t value; }") local uint32box = ffi.typeof("struct { uint32_t value; }") local uint64box = ffi.typeof("struct { uint64_t value; }") local decimal64box = ffi.typeof("struct { double value; }") +local booleanbox = ffi.typeof("struct { bool value; }") Leaf = {} function Leaf.new(base, path, src) @@ -48,6 +48,11 @@ function Leaf.new(base, path, src) for _, v in pairs(typeinfo.enum) do self.enums[v.argument] = v.argument end + elseif self.type == "union" and typeinfo.type then + self.types = {} + for _, v in pairs(typeinfo.type) do + self.types[#self.types + 1] = v.argument + end end end if src.description then @@ -97,6 +102,8 @@ function Leaf.new(base, path, src) return self end +function Leaf:get_type() return "leaf" end + function Leaf:error(msg, ...) local path = getmetatable(self).path error(("%s: %s"):format(path, msg:format(...))) @@ -106,7 +113,7 @@ function Leaf:validate_schema(schema) local cardinality = {config={0,1}, default={0,1}, description={0,1}, mandatory={0,1}, reference={0,1}, status={0,1}, type={1,1}, units={0,1}, when={0,1}} - validation.cardinality("leaf", getmetatable(self).path, cardinality, schema) + helpers.cardinality("leaf", getmetatable(self).path, cardinality, schema) end function Leaf:provide_box(leaf_type, statements) @@ -135,20 +142,32 @@ function Leaf:provide_box(leaf_type, statements) elseif leaf_type == "string" then box = {} elseif leaf_type == "boolean" then - box = {} + box = booleanbox() elseif leaf_type == "enumeration" then box = helpers.Enum.new(self.enums) elseif leaf_type == "union" then - box = helpers.Union.new(statements) + box = helpers.Union.new(self.types) elseif leaf_type == "inet:ipv4-address" then box = helpers.IPv4Box.new() + if self.default then box.value = self.default end elseif leaf_type == "inet:ipv6-address" then box = helpers.IPv6Box.new() + if self.default then box.value = self.default end + elseif leaf_type == "inet:ipv4-prefix" then + box = helpers.IPv4PrefixBox.new() + elseif leaf_type == "inet:ipv6-prefix" then + box = helpers.IPv6PrefixBox.new() + elseif leaf_type == "yang:zero-based-counter64" then + -- TODO: this can and should be done via support of typedef. + box = uint64box(0) else local path = self and getmetatable(self).path or "" - error(("Unknown type '%s' for leaf"):format(path, leaf_type)) + error(("%s: Unknown type '%s' for leaf"):format(path, leaf_type)) end + -- If there is default we should set it. + if self.default ~= nil then box.value = self.default end + return box end @@ -175,29 +194,34 @@ function Feature.new(base, path, src) return self end +function Feature:get_type() return "feature" end + function Feature:validate_schema(src) local cardinality = {description={0,1}, status={0,1}, refernece={0,1}} - validation.cardinality("feature", getmetatable(self).path, cardinality, src) + helpers.cardinality("feature", getmetatable(self).path, cardinality, src) end -- Yang list local List = {} function List.new(base, path, src) - local self = setmetatable({}, {__index=List, path=path}) + local ret = {leaves={}} + local self = setmetatable(ret, {__index=List, path=path}) self:validate_schema(src) base:add_cache(path, self) if src.key then self.key = src.key[1].argument end + if src.uses then self.uses = src.uses[1].argument end if src.leaf then for _, leaf in pairs(src.leaf) do local path = self.path.."."..leaf.argument - self[leaf.argument] = Leaf.new(base, path, leaf.statements) + self.leaves[leaf.argument] = Leaf.new(base, path, leaf.statements) end end return self end +function List:get_type() return "list" end function List:validate_schema(src) local cardinality = {config={0,1}, description={0,1}, key={0,1}, @@ -205,7 +229,7 @@ function List:validate_schema(src) cardinality["max-elements"] = {0,1} cardinality["min-elements"] = {0,1} cardinality["ordered-by"] = {0,1} - validation.cardinality("list", getmetatable(self).path, cardinality, src) + helpers.cardinality("list", getmetatable(self).path, cardinality, src) end -- Yang group @@ -237,15 +261,16 @@ function Grouping.new(base, path, src) return self end +function Grouping:get_type() return "grouping" end function Grouping:validate_schema(src) local cardinality = {description={0,1}, reference={0,1}, status={0,1}} - validation.cardinality("grouping", getmetatable(self).path, cardinality, src) + helpers.cardinality("grouping", getmetatable(self).path, cardinality, src) end local Container = {} function Container.new(base, path, src) - local ret = {leaves={}, containers={}} + local ret = {leaves={}, containers={}, lists={}} local self = setmetatable(ret, {__index=Container, path=path}) self:validate_schema(src) @@ -258,34 +283,47 @@ function Container.new(base, path, src) -- Leaf statements if src.leaf then for _, leaf in pairs(src.leaf) do - local leaf_path = path.."."..leaf.argument - self.leaves[leaf.argument] = Leaf.new(base, leaf_path, leaf.statements) + self.leaves[leaf.argument] = Leaf.new( + base, + path.."."..leaf.argument, + leaf.statements + ) end end -- Include other containers if src.container then for _, container in pairs(src.container) do - local container_path = path.."."..container.argument self.containers[container.argument] = Container.new( base, - container_path, + path.."."..container.argument, container.statements ) end end + if src.list then + for _, list in pairs(src.list) do + self.lists[list.argument] = List.new( + base, + path.."."..list.argument, + list.statements + ) + end + end + if src.uses then self.uses = src.uses[1].argument end return self end +function Container:get_type() return "container" end function Container:validate_schema(src) local cardinality = {config={0,1}, description={0,1}, presense={0,1}, reference={0,1}, status={0,1}, when={0,1}} - validation.cardinality("container", getmetatable(self).path, cardinality, src) + helpers.cardinality("container", getmetatable(self).path, cardinality, src) end -- Yang Revision @@ -305,10 +343,11 @@ function Revision.new(base, path, src) end return self end +function Revision:get_type() return "revision" end function Revision:validate_schema(src) local cardinality = {description={0,1}, reference={0,1}} - validation.cardinality("revision", getmetatable(self).path, cardinality, src) + helpers.cardinality("revision", getmetatable(self).path, cardinality, src) end -- Yang Module @@ -386,6 +425,7 @@ function Module.new(base, name, src) end return self end +function Module:get_type() return "module" end function Module:load() -- TODO: Find the file and load it. @@ -395,146 +435,71 @@ function Module:validate_schema(src) local cardinality = {contact={0,1}, description={0,1}, namespace={1,1}, organization={0,1}, prefix={1,1}, reference={0,1}} cardinality["yang-version"] = {0,1} - validation.cardinality("module", getmetatable(self).path, cardinality, src) + helpers.cardinality("module", getmetatable(self).path, cardinality, src) end function selftest() - local test_schema = [[module ietf-softwire { - namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; - prefix "softwire"; + local test_schema = [[module fruit { + namespace "urn:testing:fruit"; + prefix "fruit"; import ietf-inet-types {prefix inet; } import ietf-yang-types {prefix yang; } - organization "Softwire Working Group"; - - contact - " - Qi Sun sunqi.ietf@gmail.com - Hao Wang wangh13@mails.tsinghua.edu.cn - Yong Cui yong@csnet1.cs.tsinghua.edu.cn - Ian Farrer ian.farrer@telekom.de - Mohamed Boucadair mohamed.boucadair@orange.com - Rajiv Asati rajiva@cisco.com - "; - - description - "This document defines a YANG data model for the configuration and - management of IPv4-in-IPv6 Softwire Border Routers and Customer - Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T - Softwire mechanisms. - - Copyright (c) 2014 IETF Trust and the persons identified - as authors of the code. All rights reserved. - This version of this YANG module is part of RFC XXX; see the RFC - itself for full legal notices."; - - revision 2015-09-30 { - description - "Version-04: fix YANG syntax; Add flags to map-rule; Remove - the map-rule-type element. "; - - reference "tbc"; - } - - revision 2015-04-07 { - description - "Version-03: Integrate lw4over6; Updata state nodes; Correct - grammar errors; Reuse groupings; Update descriptions. - Simplify the model."; + organization "Fruit Inc."; - reference "tbc"; - } - - revision 2015-02-10 { - description "Version-02: Add notifications."; - reference "tbc"; - } + contact "John Smith fake@person.tld"; - revision 2015-02-06 { - description - "Version-01: Correct grammar errors; Reuse groupings; Update - descriptions."; + description "Module to test YANG schema lib"; + revision 2016-05-27 { + description "Revision 1"; reference "tbc"; } - revision 2015-02-02 { - description "Initial revision."; + revision 2016-05-28 { + description "Revision 2"; reference "tbc"; } - feature lw4over6 { - description - "Lightweight 4over6 moves the Network Address and Port - Translation (NAPT) function from the centralized DS-Lite tunnel - concentrator to the tunnel client located in the Customer - Premises Equipment (CPE). This removes the requirement for a - Carrier Grade NAT function in the tunnel concentrator and - reduces the amount of centralized state that must be held to a - per-subscriber level. In order to delegate the NAPT function - and make IPv4 Address sharing possible, port-restricted IPv4 - addresses are allocated to the CPEs."; - - reference "I-D.ietf-softwire-lw4over6"; + feature bowl { + description "A fruit bowl"; + reference "fruit-bowl"; } - feature map-e { - description - "MAP-E is a mechanism for transporting IPv4 packets across an - IPv6 network using IP encapsulation, and a generic mechanism - for mapping between IPv6 addresses and IPv4 addresses and - transport layer ports."; - - reference "I-D.ietf-softwire-map"; - } + grouping fruit { + description "Represets a piece of fruit"; - grouping port-set { - description - "Use the PSID algorithm to represent a range of transport layer - ports."; - - leaf offset { - type uint8 { - range 0..16; - } - mandatory true; - description - "The number of offset bits. In Lightweight 4over6, the default - value is 0 for assigning one contiguous port range. In MAP-E/T, - the default value is 6, which excludes system ports by default - and assigns distributed port ranges. If the this parameter is - larger than 0, the value of offset MUST be greater than 0."; - } - - leaf psid { - type uint16; + leaf name { + type string; mandatory true; - description - "Port Set Identifier (PSID) value, which identifies a set - of ports algorithmically."; + description "Name of fruit."; } - leaf psid-len { + leaf score { type uint8 { - range 0..16; + range 0..10; } mandatory true; - description - "The length of PSID, representing the sharing ratio for an - IPv4 address."; + description "How nice is it out of 10"; + } + + leaf tree-grown { + type boolean; + description "Is it grown on a tree?"; } } - container softwire-config { - description - "The configuration data for Softwire instances. And the shared - data describes the softwire data model which is common to all of - the different softwire mechanisms, such as description."; + container fruit-bowl { + description "Represents a fruit bowl"; leaf description { type string; - description "A textual description of Softwire."; + description "About the bowl"; + } + + list contents { + uses fruit; } } }]] @@ -553,42 +518,40 @@ function selftest() local mod = Module.new(base, schema.module[1].argument, schema.module[1].statements) - assert(mod.name == "ietf-softwire") - assert(mod.namespace == "urn:ietf:params:xml:ns:yang:ietf-softwire") - assert(mod.prefix == "softwire") - assert(mod.contact) - assert(mod.organization) - assert(mod.description) + assert(mod.name == "fruit") + assert(mod.namespace == "urn:testing:fruit") + assert(mod.prefix == "fruit") + assert(mod.contact == "John Smith fake@person.tld") + assert(mod.organization == "Fruit Inc.") + assert(mod.description == "Module to test YANG schema lib") -- Check both modules exist. (Also need to check they've loaded) assert(mod.modules["ietf-inet-types"]) assert(mod.modules["ietf-yang-types"]) -- Check all revisions are accounted for. - assert(mod.revisions["2015-02-02"].description) - assert(mod.revisions["2015-02-06"].description) - assert(mod.revisions["2015-02-10"].description) - assert(mod.revisions["2015-04-07"].description) - assert(mod.revisions["2015-09-30"].description) + assert(mod.revisions["2016-05-27"].description == "Revision 1") + assert(mod.revisions["2016-05-28"].description == "Revision 2") -- Check that the feature statements are there. - assert(mod.features["lw4over6"].description) - assert(mod.features["lw4over6"].reference) - assert(mod.features["map-e"].description) - assert(mod.features["map-e"].reference) + assert(mod.features["bowl"].description) -- Check the groupings - assert(mod.groupings["port-set"]) - assert(mod.groupings["port-set"].description) - assert(mod.groupings["port-set"].leaves["offset"]) - assert(mod.groupings["port-set"].leaves["offset"].type == "uint8") - assert(mod.groupings["port-set"].leaves["psid-len"].mandatory == true) - assert(mod.groupings["port-set"].leaves["psid-len"].range[1] == 0) - assert(mod.groupings["port-set"].leaves["psid-len"].range[2] == 16) + assert(mod.groupings["fruit"]) + assert(mod.groupings["fruit"].description) + assert(mod.groupings["fruit"].leaves["name"].type == "string") + assert(mod.groupings["fruit"].leaves["name"].mandatory == true) + assert(mod.groupings["fruit"].leaves["name"].description == "Name of fruit.") + assert(mod.groupings["fruit"].leaves["score"].type == "uint8") + assert(mod.groupings["fruit"].leaves["score"].mandatory == true) + assert(mod.groupings["fruit"].leaves["score"].range[1] == 0) + assert(mod.groupings["fruit"].leaves["score"].range[2] == 10) -- Check the containers description (NOT the leaf called "description") - assert(type(mod.containers["softwire-config"].description) == "string") + assert(mod.containers["fruit-bowl"].description == "Represents a fruit bowl") -- Check the container has a leaf called "description" - assert(mod.containers["softwire-config"].leaves.description) + local desc = mod.containers["fruit-bowl"].leaves.description + assert(desc.type == "string") + assert(desc.description == "About the bowl") end \ No newline at end of file diff --git a/src/lib/yang/validation.lua b/src/lib/yang/validation.lua deleted file mode 100644 index a699fc5269..0000000000 --- a/src/lib/yang/validation.lua +++ /dev/null @@ -1,17 +0,0 @@ -module(..., package.seeall) - -function cardinality(kw, path, statements, haystack) - for s, c in pairs(statements) do - if (c[1] >= 1 and (not haystack[s])) or - (#statements[s] < c[1] and #statements[s] > c[2]) then - if c[1] == c[2] then - error(("Expected %d %s statement(s) in %s:%s"):format( - c[1], s, kw, path)) - else - local err = "Expected between %d and %d of %s statement(s) in %s:%s" - error((err):format(c[1], c[2], s, kw, path)) - end - end - end - return true -end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 25f563bb5a..247b6cb166 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -39,6 +39,8 @@ local schema = require("lib.yang.schema") local helpers = require("lib.yang.helpers") local parser = require("lib.yang.parser") local Container = helpers.Container +local asserterror = helpers.asserterror +local setvalue = helpers.setvalue Base = {} function Base.new(filename) @@ -64,9 +66,7 @@ function Base:load(src) ) self.schema[src.module[1].argument] = mod - -- TODO: don't use rawget here. - local data_root = rawget(self.data, "root") - data_root[mod.name] = Container.new(self, mod.name) + self.data:add_container(mod.name) return self.schema end @@ -112,9 +112,7 @@ function Base:produce_data_tree(schema_node, data_node) local new_path = path.."."..name local new_node = Container.new(self, new_path) - -- TODO: change me, we shouldn't be using rawget here! - local root = rawget(data_node, "root") - root[name] = new_node + data_node:add_to_root(name, new_node) schema_node = self:get_schema(new_path) data_node = new_node self:produce_data_tree(schema_node, new_node) @@ -122,33 +120,57 @@ function Base:produce_data_tree(schema_node, data_node) -- If the container has a "uses" statement we must copy across the -- leaves from the container it references to this container. if container.uses then - local root = rawget(data_node, "root") - local grouping = self:get_module().groupings[container.uses] - if not grouping then - self:error(path, name, "Cannot find grouping '%s'.", container.uses) - end - - -- Copy. - for name, leaf in pairs(grouping.leaves) do - -- We also need to register the schema node at the new path - local grouping_path = new_path.."."..name - self:add_cache(grouping_path, leaf) - root[name] = leaf:provide_box() - end + self:handle_use(container, data_node, path, name) end end end if schema_node.leaves then for name, leaf in pairs(schema_node.leaves) do - -- TODO remove the use of rawget here, it's not good. - local root = rawget(data_node, "root") - root[name] = leaf:provide_box() + data_node:add_to_root(name, leaf:provide_box()) end end + + if schema_node.lists then + for name, list in pairs(schema_node.lists) do + local list_path = path.."."..name + local container = Container.new(self, list_path) + data_node:add_to_root(name, container) + + if list.uses then + local template = Container.new(self, list_path) + self:handle_use(list, template, list_path, name) + container:set_template(template) + end + end + end + return self.data end +function Base:schema_for_uses(schema) + if schema.uses == nil then + error("Can only find schema for a node which uses the `use` statement.") + end + + return self:get_module().groupings[schema.uses] +end + +function Base:handle_use(schema_node, data_node, path, name) + local grouping = self:schema_for_uses(schema_node) + if not grouping then + self:error(path, name, "Cannot find grouping '%s'.", schema_node.uses) + end + + -- Copy. + for name, leaf in pairs(grouping.leaves) do + -- We also need to register the schema node at the new path + local grouping_path = path.."."..name + self:add_cache(grouping_path, leaf) + data_node:add_to_root(name, leaf:provide_box()) + end +end + function Base:add(key, node) self.data[key] = node end @@ -168,212 +190,96 @@ function load_schema_file(filename) end function selftest() - local test_yang = [[module ietf-softwire { - namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; - prefix "softwire"; + local test_schema = [[module fruit { + namespace "urn:testing:fruit"; + prefix "fruit"; import ietf-inet-types {prefix inet; } import ietf-yang-types {prefix yang; } - organization "Softwire Working Group"; - - contact - " - Qi Sun sunqi.ietf@gmail.com - Hao Wang wangh13@mails.tsinghua.edu.cn - Yong Cui yong@csnet1.cs.tsinghua.edu.cn - Ian Farrer ian.farrer@telekom.de - Mohamed Boucadair mohamed.boucadair@orange.com - Rajiv Asati rajiva@cisco.com - "; - - description - "This document defines a YANG data model for the configuration and - management of IPv4-in-IPv6 Softwire Border Routers and Customer - Premises Equipment. It covers Lightweight 4over6, MAP-E and MAP-T - Softwire mechanisms. - - Copyright (c) 2014 IETF Trust and the persons identified - as authors of the code. All rights reserved. - This version of this YANG module is part of RFC XXX; see the RFC - itself for full legal notices."; - - - revision 2015-09-30 { - description - "Version-04: fix YANG syntax; Add flags to map-rule; Remove - the map-rule-type element. "; - - reference "tbc"; - } + organization "Fruit Inc."; - revision 2015-04-07 { - description - "Version-03: Integrate lw4over6; Updata state nodes; Correct - grammar errors; Reuse groupings; Update descriptions. - Simplify the model."; + contact "John Smith fake@person.tld"; - reference "tbc"; - } + description "Module to test YANG schema lib"; - revision 2015-02-10 { - description "Version-02: Add notifications."; + revision 2016-05-27 { + description "Revision 1"; reference "tbc"; } - revision 2015-02-06 { - description - "Version-01: Correct grammar errors; Reuse groupings; Update - descriptions."; - + revision 2016-05-28 { + description "Revision 2"; reference "tbc"; } - revision 2015-02-02 { - description "Initial revision."; - reference "tbc"; + feature bowl { + description "A fruit bowl"; + reference "fruit-bowl"; } - feature lw4over6 { - description - "Lightweight 4over6 moves the Network Address and Port - Translation (NAPT) function from the centralized DS-Lite tunnel - concentrator to the tunnel client located in the Customer - Premises Equipment (CPE). This removes the requirement for a - Carrier Grade NAT function in the tunnel concentrator and - reduces the amount of centralized state that must be held to a - per-subscriber level. In order to delegate the NAPT function - and make IPv4 Address sharing possible, port-restricted IPv4 - addresses are allocated to the CPEs."; - reference "I-D.ietf-softwire-lw4over6"; - } + grouping fruit { + description "Represets a piece of fruit"; - feature map-e { - description - "MAP-E is a mechanism for transporting IPv4 packets across an - IPv6 network using IP encapsulation, and a generic mechanism - for mapping between IPv6 addresses and IPv4 addresses and - transport layer ports."; - - reference "I-D.ietf-softwire-map"; - } - - grouping port-set { - description - "Use the PSID algorithm to represent a range of transport layer - ports."; - - leaf offset { - type uint8 { - range 0..16; + leaf name { + type string; + mandatory true; + description "Name of fruit."; } - mandatory true; - description - "The number of offset bits. In Lightweight 4over6, the defaul - value is 0 for assigning one contiguous port range. In MAP-E/T, - the default value is 6, which excludes system ports by default - and assigns distributed port ranges. If the this parameter is - larger than 0, the value of offset MUST be greater than 0."; - } - - leaf psid { - type uint16; - mandatory true; - description - "Port Set Identifier (PSID) value, which identifies a set - of ports algorithmically."; - } - leaf psid-len { - type uint8 { - range 0..16; - } - mandatory true; - description - "The length of PSID, representing the sharing ratio for an - IPv4 address."; - } - } - - grouping binding-entry { - description - "The lwAFTR maintains an address binding table that contains - thebinding between the lwB4's IPv6 address, the allocated IPv4 - address and restricted port-set."; - - leaf binding-ipv6info { - type union { - type inet:ipv6-address; - type inet:ipv6-prefix; + leaf score { + type uint8 { + range 0..10; + } + mandatory true; + description "How nice is it out of 10"; } - mandatory true; - description - "The IPv6 information for a binding entry. - If it's an IPv6 prefix, it indicates that - the IPv6 source address of the lwB4 is constructed - according to the description in RFC7596; - if it's an IPv6 address, it means the lwB4 uses"; + leaf tree-grown { + type boolean; + description "Is it grown on a tree?"; + } } - leaf binding-ipv4-addr { - type inet:ipv4-address; - mandatory true; - description - "The IPv4 address assigned to the lwB4, which is - used as the IPv4 external address - for lwB4 local NAPT44."; - } + container fruit-bowl { + description "Represents a fruit bowl"; - container port-set { - description - "For Lightweight 4over6, the default value - of offset should be 0, to configure one contiguous - port range."; - uses port-set { - refine offset { - default "0"; - } + leaf description { + type string; + description "About the bowl"; } - } - leaf lwaftr-ipv6-addr { - type inet:ipv6-address; - mandatory true; - description "The IPv6 address for lwaftr."; - } - - leaf lifetime { - type uint32; - units seconds; - description "The lifetime for the binding entry"; + list contents { + uses fruit; + } } - } + }]] + local base = load_schema(test_schema) + local data = base:produce_data_tree() + local bowl = data.fruit["fruit-bowl"] + bowl.description = "hello!" + assert(bowl.description == "hello!") - container softwire-config { - description - "The configuration data for Softwire instances. And the shared - data describes the softwire data model which is common to all of - the different softwire mechanisms, such as description."; + -- Add items to fruit-bowl's contents list. + bowl.contents:add_item({name="Banana", score=10}) + assert(bowl.contents[1].name == "Banana") + assert(bowl.contents[1].score == 10) - leaf description { - type string; - description "A textual description of Softwire."; - } + -- Check that validation still works in lists with groupings. + asserterror(setvalue, bowl.contents[1].score, "fail") - container testgroup { - uses binding-entry; - } - } -}]] - local base = load_schema(test_yang) - local data = base:produce_data_tree() + -- Check that an entry can't be added with missing required fields + asserterror(bowl.contents.add_item, bowl.contents, {score=10}) - data["ietf-softwire"]["softwire-config"].description = "hello!" - assert(data["ietf-softwire"]["softwire-config"].description == "hello!") + -- Check that an entry with incorrect data can't be added. + asserterror( + bowl.contents.add_item, + bowl.contents, + {name="Pear", score=5, invalid=true} + ) - -- Test grouping. - data["ietf-softwire"]["softwire-config"].testgroup["lifetime"] = 72 - assert(data["ietf-softwire"]["softwire-config"].testgroup["lifetime"] == 72) + -- Finally check tht validation occurs when you're adding entries with + -- invalid data in them, in this case sore needs to be an integer. + asserterror(bowl.contents, bowl.contents, {name="Pear", score="Good"}) end \ No newline at end of file From 263c0e98d7568a446cb702d9f89cd6b1074a3541 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 28 Jun 2016 11:50:27 +0000 Subject: [PATCH 019/631] Fix problems brought up in review - Removes the duplicate boxed IPv4 and IPv6 values - Removes unused variables - Refactor some code to clean up and prevent cyclic importing - Move types into a table for faster and cleaner code than if else blocks - Fix some other bugs found during testing --- src/lib/protocol/ipv4.lua | 8 ++ src/lib/protocol/ipv6.lua | 4 + src/lib/yang/helpers.lua | 248 +++++++++++++++++++++++--------------- src/lib/yang/schema.lua | 73 +---------- src/lib/yang/yang.lua | 4 +- 5 files changed, 167 insertions(+), 170 deletions(-) diff --git a/src/lib/protocol/ipv4.lua b/src/lib/protocol/ipv4.lua index a627b5546b..24a0b7f52b 100644 --- a/src/lib/protocol/ipv4.lua +++ b/src/lib/protocol/ipv4.lua @@ -89,8 +89,16 @@ function ipv4:ntop (n) return ffi.string(c_str) end +function ipv4:set(addr) + return ipv4:pton(addr) +end + -- Instance methods +function ipv4:get() + return ipv4:ntop(self) +end + function ipv4:version (v) return lib.bitfield(16, self:header(), 'ihl_v_tos', 0, 4, v) end diff --git a/src/lib/protocol/ipv6.lua b/src/lib/protocol/ipv6.lua index 7db73c8288..2648331294 100644 --- a/src/lib/protocol/ipv6.lua +++ b/src/lib/protocol/ipv6.lua @@ -96,6 +96,10 @@ function ipv6:ntop (n) return ffi.string(c_str) end +function ipv6:set(addr) + self:pton(addr) +end + -- Construct the solicited-node multicast address from the given -- unicast address by appending the last 24 bits to ff02::1:ff00:0/104 function ipv6:solicited_node_mcast (n) diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index 5011d0dd06..f53715eac1 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -2,17 +2,75 @@ module(..., package.seeall) local h = require("syscall.helpers") +local ffi = require("ffi") local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") local corelib = require("core.lib") +-- Create a small wrapper around the FFI types to provde get and set +local FFITypeWrapper = {} +function FFITypeWrapper.new(ffibox) + return setmetatable({box=ffibox()}, { + __index=FFITypeWrapper, + __call=function(t, v) + local wrappedbox = t.new(ffibox) + if v ~= nil then wrappedbox:set(v) end + return wrappedbox + end, + }) +end +function FFITypeWrapper:set(value) + self.box.value = value +end +function FFITypeWrapper:get() + return self.box.value +end + +-- Use ffi types because they will validate that numeric values are being +-- provided. The downside is that integer overflow could occur on these. This +-- route has been selected as validation will be faster than attempting to +-- validate in Lua. +local box_types = { + int8 = FFITypeWrapper.new(ffi.typeof("struct { int8_t value; }")), + int16 = FFITypeWrapper.new(ffi.typeof("struct { int16_t value; }")), + int32 = FFITypeWrapper.new(ffi.typeof("struct { int32_t value; }")), + int64 = FFITypeWrapper.new(ffi.typeof("struct { int64_t value; }")), + uint8 = FFITypeWrapper.new(ffi.typeof("struct { uint8_t value; }")), + uint16 = FFITypeWrapper.new(ffi.typeof("struct { uint16_t value; }")), + uint32 = FFITypeWrapper.new(ffi.typeof("struct { uint32_t value; }")), + uint64 = FFITypeWrapper.new(ffi.typeof("struct { uint64_t value; }")), + decimal64 = FFITypeWrapper.new(ffi.typeof("struct { double value; }")), + boolean = FFITypeWrapper.new(ffi.typeof("struct { bool value; }")), +} +-- Add inet types, found: https://tools.ietf.org/html/rfc6021 +-- Should be done via yang module import but using ad-hoc method for now. +box_types["inet:ipv4-address"] = ipv4.pton +box_types["inet:ipv6-address"] = ipv6.pton +box_types["yang:zero-based-counter64"] = function () + return box_types.uint64(0) +end + +function create_box(leaf_type, default) + local box = assert(box_types[leaf_type], "Unsupported type: "..leaf_type) + if box and default ~= nil then + box = box(default) + elseif box then + box = box() + end + return box +end + function extract_nodes(schema) + -- This function takes a table which is in the format: + -- {1={keyword="leaf", statements={...}}} + -- and converts this to a more useful easy to access: + -- {leaf={1={...}}} + local nodes = {} for _, v in pairs(schema) do - -- Recursively apply this + -- Node has statements (children) so we should recursively apply the + -- `extract_nodes` function to them so the entire tree is extracted if v.statements then v.statements = extract_nodes(v.statements) end - - -- Add to the nodes table. if nodes[v.keyword] then table.insert(nodes[v.keyword], v) else @@ -22,124 +80,123 @@ function extract_nodes(schema) return nodes end -IPv4Box = {} -function IPv4Box.new() - local ret = {root={}} - return setmetatable(ret, { - __index = function (t, k) return rawget(ret, "root").value end, - __newindex = function(t, k, v) - local ipv4boxed, err = ipv4:pton(v) - if ipv4boxed == false then error(err) end - rawget(ret, "root").value = ipv4boxed - end - }) +local StringBox = {} +function StringBox.new() + return setmetatable({}, {__index=StringBox}) end -IPv6Box = {} -function IPv6Box.new() - local ret = {root={}} - return setmetatable(ret, { - __index = function (t, k) return rawget(ret, "root").value end, - __newindex = function(t, k, v) - local ipv6boxed, err = ipv6:pton(v) - if ipv6boxed == false then error(err) end - rawget(ret, "root").value = ipv6boxed - end - }) +function StringBox:get() + return self.value +end + +function StringBox:set(value) + self.value = value end +box_types["string"] = StringBox.new -IPv4PrefixBox = {} +local IPv4PrefixBox = {} function IPv4PrefixBox.new() local ret = {root={}} - return setmetatable(ret, { - __newindex = function (t, k, v) - -- split the netmask and prefix up. - local raw = h.split("/", v) - local addr, err = ipv4:pton(raw[1]) - if addr == false then error(err) end - local prefix = tonumber(raw[2]) - if prefix > 32 or prefix < 0 then - error(("The prefix length is invalid. (%s)"):format(raw[2])) - end - rawget(ret, "root").value = {addr, prefix} - end, - __index = function (t, k, v) return rawget(ret, "root").value end - }) + return setmetatable(ret, {__index=IPv4PrefixBox}) end -IPv6PrefixBox = {} +function IPv4PrefixBox:get() + return self.value +end +box_types["inet:ipv4-prefix"] = IPv4PrefixBox.new + +function IPv4PrefixBox:set(value) + -- split the netmask and prefix up. + local raw = h.split("/", value) + local addr, err = ipv4:pton(raw[1]) + if addr == false then error(err) end + local prefix = tonumber(raw[2]) + if prefix > 32 or prefix < 0 then + error(("The prefix length is invalid. (%s)"):format(raw[2])) + end + self.value = {addr, prefix} +end + +local IPv6PrefixBox = {} function IPv6PrefixBox.new() local ret = {root={}} - return setmetatable(ret, { - __newindex = function (t, k, v) - -- split the netmask and prefix up. - local raw = h.split("/", v) - local addr, err = ipv6:pton(raw[1]) - if addr == false then error(err) end - local prefix = tonumber(raw[2]) - if prefix > 128 or prefix < 0 then - error(("The prefix length is invalid. (%s)"):format(raw[2])) - end - rawget(ret, "root").value = {addr, prefix} - end, - __index = function (t, k, v) return rawget(ret, "root").value end - }) + return setmetatable(ret, {__index=IPv6PrefixBox}) +end +function IPv6PrefixBox:get() + return self.value +end +function IPv6PrefixBox:set(value) + -- split the netmask and prefix up. + local raw = h.split("/", value) + local addr, err = ipv6:pton(raw[1]) + if addr == false then error(err) end + local prefix = tonumber(raw[2]) + if prefix > 128 or prefix < 0 then + error(("The prefix length is invalid. (%s)"):format(raw[2])) + end + self.value = {addr, prefix} end +box_types["inet:ipv6-prefix"] = IPv6PrefixBox.new -Enum = {} +local Enum = {} function Enum.new(options) - local ret = {root={}} local opts = {} for _, option in pairs(options) do opts[option] = option end - return setmetatable(ret, { - __index = function (t, k) return rawget(ret, "root").value end, - __newindex = function(t, k, v) - if opts[v] then rawget(ret, "root").value = v - else error("Value "..v.." is not a valid option for this Enum.") end - end - }) + return setmetatable({options=opts}, {__index=Enum}) end -Union = {} +function Enum:get() + return self.value +end + +function Enum:set(value) + if self.options[value] then + self.value = value + else + error("Value "..value.." is not a valid option for this Enum.") + end +end +box_types["enumeration"] = Enum.new + +local Union = {} function Union.new(types) - local ret = {root={}, types={}} + local ret = {types={}} - -- Importing here to prevent cyclic imports. - local Leaf = require("lib.yang.schema").Leaf for _, name in pairs(types) do -- 9.12 specifies unions cannot contain "leafref" or "empty" if name == "empty" or name == "leafref" then error("Union type cannot contain 'empty' or 'leafref'") end - ret.types[name] = Leaf:provide_box(name) + ret.types[name] = create_box(name) end - return setmetatable(ret, { - __index = function (t, k) - return rawget(ret, "root").box.value - end, - __newindex = function(t, k, v) - local function setbox(dest, v) dest.value = v end - local root = rawget(ret, "root") - for name, box in pairs(rawget(ret, "types")) do - local valid = pcall(setbox, box, v) - if valid then - root.box = box - return -- don't want to continue. - end - end - error(("Unable to find matching type for '%s' (%s)"):format(v, type(v))) + return setmetatable(ret, {__index=Union}) +end + +function Union:get() + return self.value +end + +function Union:set(v) + local function setbox(dest, val) dest.value = val end + for _, box in pairs(self.types) do + local valid = pcall(setbox, box, v) + if valid then + self.box = box + return -- don't want to continue. end - }) + end + error(("Unable to find matching type for '%s' (%s)"):format(v, type(v))) end +box_types["union"] = Union.new Container = {} function Container.new(base, path) local ret = {root={}, base=base, path=path} return setmetatable(ret, { - __newindex = function (t, k, v) + __newindex = function (_, k, v) -- Validate the value prior to setting it. local node_path = ret.path.."."..k local schema = assert( @@ -152,21 +209,20 @@ function Container.new(base, path) validation(v) end end - local box = assert(ret.root[k], "Unknown leaf '"..node_path.."'") - box.value = v + box:set(v) end, __index = function(t, k) - local table = rawget(t, "root") - local prop = table[k] + local root = rawget(t, "root") + local prop = root[k] if not prop then -- This could be because it's a method defined on Container. return Container[k] - elseif prop.value == nil then + elseif prop.get == nil then return prop else - return prop.value + return prop:get() end end, }) @@ -175,7 +231,7 @@ end function Container:set_template(template) rawset(self, "template", template) end -function Container:get_template(template) +function Container:get_template() return rawget(self, "template") end @@ -207,7 +263,7 @@ function Container:add_item(item) for name, leaf in pairs(leaves) do data[name] = leaf end -- Add the data to the container. - for name, leaf in pairs(data) do + for name, _ in pairs(data) do if leaves[name] == nil then error("Unknown field in list: '"..name.."'") elseif item[name] == nil and leaves[name].mandatory then @@ -283,13 +339,13 @@ function selftest() asserterror(setvalue, con, "testbox", "should fail") -- Test the IPv4 box (both valid and invalid data). - root.testbox = IPv4Box.new() + root.testbox = box_types["inet:ipv4-address"]() con.testbox = "8.8.8.8" assert(ipv4:ntop(con.testbox) == "8.8.8.8") asserterror(setvalue, con, "testbox", "256.8.8.8") -- Test the IPv6 box (both valid and invalid data). - root.testbox = IPv6Box.new() + root.testbox = box_types["inet:ipv6-address"]() con.testbox = "::1" assert(ipv6:ntop(con.testbox) == "::1") asserterror(setvalue, con, "testbox", "not ipv6") diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 2d82f33469..81afd79f9f 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -13,22 +13,6 @@ module(..., package.seeall) local helpers = require("lib.yang.helpers") local h = require("syscall.helpers") --- Use ffi types because they will validate that numeric values are being --- provided. The downside is that integer overflow could occur on these. This --- route has been selected as validation will be faster than attempting to --- validate in Lua. -local ffi = require("ffi") -local int8box = ffi.typeof("struct { int8_t value; }") -local int16box = ffi.typeof("struct { int16_t value; }") -local int32box = ffi.typeof("struct { int32_t value; }") -local int64box = ffi.typeof("struct { int64_t value; }") -local uint8box = ffi.typeof("struct { uint8_t value; }") -local uint16box = ffi.typeof("struct { uint16_t value; }") -local uint32box = ffi.typeof("struct { uint32_t value; }") -local uint64box = ffi.typeof("struct { uint64_t value; }") -local decimal64box = ffi.typeof("struct { double value; }") -local booleanbox = ffi.typeof("struct { bool value; }") - Leaf = {} function Leaf.new(base, path, src) local self = setmetatable({}, {__index=Leaf, path=path}) @@ -87,7 +71,7 @@ function Leaf.new(base, path, src) if not self.validation then self.validation = {} end self.validation[#self.validation + 1] = function(v) if v < self.range[1] or v > self.range[2] then - self:error("Value '%s' is out of range", path, value) + self:error("Value '%s' is out of range", path, v) end end end @@ -116,61 +100,6 @@ function Leaf:validate_schema(schema) helpers.cardinality("leaf", getmetatable(self).path, cardinality, schema) end -function Leaf:provide_box(leaf_type, statements) - local box - - if not leaf_type then leaf_type = self.type end - - if leaf_type == "int8" then - box = int8box() - elseif leaf_type == "int16" then - box = int16box() - elseif leaf_type == "int32" then - box = int32box() - elseif leaf_type == "int64" then - box = int64box() - elseif leaf_type == "uint8" then - box = uint8box() - elseif leaf_type == "uint16" then - box = uint16box() - elseif leaf_type == "uint32" then - box = uint32box() - elseif leaf_type == "uint64" then - box = uint64box() - elseif leaf_type == "decimal64" then - box = decimal64box() - elseif leaf_type == "string" then - box = {} - elseif leaf_type == "boolean" then - box = booleanbox() - elseif leaf_type == "enumeration" then - box = helpers.Enum.new(self.enums) - elseif leaf_type == "union" then - box = helpers.Union.new(self.types) - elseif leaf_type == "inet:ipv4-address" then - box = helpers.IPv4Box.new() - if self.default then box.value = self.default end - elseif leaf_type == "inet:ipv6-address" then - box = helpers.IPv6Box.new() - if self.default then box.value = self.default end - elseif leaf_type == "inet:ipv4-prefix" then - box = helpers.IPv4PrefixBox.new() - elseif leaf_type == "inet:ipv6-prefix" then - box = helpers.IPv6PrefixBox.new() - elseif leaf_type == "yang:zero-based-counter64" then - -- TODO: this can and should be done via support of typedef. - box = uint64box(0) - else - local path = self and getmetatable(self).path or "" - error(("%s: Unknown type '%s' for leaf"):format(path, leaf_type)) - end - - -- If there is default we should set it. - if self.default ~= nil then box.value = self.default end - - return box -end - -- Yang feature local Feature = {} function Feature.new(base, path, src) diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 247b6cb166..aa64b43391 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -127,7 +127,7 @@ function Base:produce_data_tree(schema_node, data_node) if schema_node.leaves then for name, leaf in pairs(schema_node.leaves) do - data_node:add_to_root(name, leaf:provide_box()) + data_node:add_to_root(name, helpers.create_box(leaf.type, leaf.defalt)) end end @@ -167,7 +167,7 @@ function Base:handle_use(schema_node, data_node, path, name) -- We also need to register the schema node at the new path local grouping_path = path.."."..name self:add_cache(grouping_path, leaf) - data_node:add_to_root(name, leaf:provide_box()) + data_node:add_to_root(name, helpers.create_box(leaf.type, leaf.default)) end end From e2b22a795048e3d8bcdaa037eceb5b3b5abe9358 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 20 Jul 2016 11:54:46 +0200 Subject: [PATCH 020/631] Append 'src/' to example.yang filepath if executing snabb from root folder --- src/lib/yang/parser.lua | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index f4031fd9f7..b0f0ef55b8 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -282,6 +282,8 @@ function parse_file(filename) end function selftest() + local S = require("syscall") + local function assert_equal(a, b) if not lib.equal(a, b) then print(a, b) @@ -329,6 +331,11 @@ function selftest() return table.concat({...}, "\n") end + local function file_exists(path) + local stat = S.stat(path) + return stat and stat.isreg + end + -- Test the string parser test_string("foo", "foo") test_string([["foo"]], "foo") @@ -355,6 +362,11 @@ function selftest() test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type"}}}}) - -- Expects tests to be run from the "src" directory at the root of the repo - parse_file("src/lib/yang/example.yang") + local filename = "lib/yang/example.yang" + local pwd = S.getcwd() + -- If current path is root append src/ to filename + if file_exists(pwd.."/src/snabb") then + filename = "src/"..filename + end + parse_file(filename) end From 90016a9fc917d248161e1f8b9c0f06d450460ae0 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 27 Jul 2016 18:01:03 +0200 Subject: [PATCH 021/631] Fix box problems & make selftest more robust - Reintroduce IPv6 box and IPv4 box to allow "nil" to be stored. - Make finding path to the yang test file more robust by allowing you to call it from any path - Add getter to IPv6 address - Fix problems with FFIType wrapper --- src/lib/protocol/ipv6.lua | 4 ++ src/lib/yang/helpers.lua | 94 +++++++++++++++++++++++++-------------- src/lib/yang/parser.lua | 22 ++++----- 3 files changed, 73 insertions(+), 47 deletions(-) diff --git a/src/lib/protocol/ipv6.lua b/src/lib/protocol/ipv6.lua index 2648331294..6c49c6156f 100644 --- a/src/lib/protocol/ipv6.lua +++ b/src/lib/protocol/ipv6.lua @@ -96,6 +96,10 @@ function ipv6:ntop (n) return ffi.string(c_str) end +function ipv6:get() + return self:ntop(self) +end + function ipv6:set(addr) self:pton(addr) end diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index f53715eac1..7b4d350c5e 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -7,45 +7,44 @@ local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") local corelib = require("core.lib") --- Create a small wrapper around the FFI types to provde get and set -local FFITypeWrapper = {} -function FFITypeWrapper.new(ffibox) - return setmetatable({box=ffibox()}, { - __index=FFITypeWrapper, - __call=function(t, v) - local wrappedbox = t.new(ffibox) - if v ~= nil then wrappedbox:set(v) end - return wrappedbox - end, - }) -end -function FFITypeWrapper:set(value) - self.box.value = value +-- Create small getter and setter wrapper for ffi structs. +local FFIType = {} + +function FFIType:set(value) + self.value = value end -function FFITypeWrapper:get() - return self.box.value + +function FFIType:get() + return self.value end + +-- This wraps the FFIType to provide it with the box name, + -- Use ffi types because they will validate that numeric values are being -- provided. The downside is that integer overflow could occur on these. This -- route has been selected as validation will be faster than attempting to -- validate in Lua. local box_types = { - int8 = FFITypeWrapper.new(ffi.typeof("struct { int8_t value; }")), - int16 = FFITypeWrapper.new(ffi.typeof("struct { int16_t value; }")), - int32 = FFITypeWrapper.new(ffi.typeof("struct { int32_t value; }")), - int64 = FFITypeWrapper.new(ffi.typeof("struct { int64_t value; }")), - uint8 = FFITypeWrapper.new(ffi.typeof("struct { uint8_t value; }")), - uint16 = FFITypeWrapper.new(ffi.typeof("struct { uint16_t value; }")), - uint32 = FFITypeWrapper.new(ffi.typeof("struct { uint32_t value; }")), - uint64 = FFITypeWrapper.new(ffi.typeof("struct { uint64_t value; }")), - decimal64 = FFITypeWrapper.new(ffi.typeof("struct { double value; }")), - boolean = FFITypeWrapper.new(ffi.typeof("struct { bool value; }")), + int8 = ffi.typeof("struct { int8_t value; }"), + int16 = ffi.typeof("struct { int16_t value; }"), + int32 = ffi.typeof("struct { int32_t value; }"), + int64 = ffi.typeof("struct { int64_t value; }"), + uint8 = ffi.typeof("struct { uint8_t value; }"), + uint16 = ffi.typeof("struct { uint16_t value; }"), + uint32 = ffi.typeof("struct { uint32_t value; }"), + uint64 = ffi.typeof("struct { uint64_t value; }"), + decimal64 = ffi.typeof("struct { double value; }"), + boolean = ffi.typeof("struct { bool value; }"), } + +-- Iterate through the boxes and set the FFIType metatype +for _, box in pairs(box_types) do + ffi.metatype(box, {__index=FFIType}) +end + -- Add inet types, found: https://tools.ietf.org/html/rfc6021 -- Should be done via yang module import but using ad-hoc method for now. -box_types["inet:ipv4-address"] = ipv4.pton -box_types["inet:ipv6-address"] = ipv6.pton box_types["yang:zero-based-counter64"] = function () return box_types.uint64(0) end @@ -94,6 +93,34 @@ function StringBox:set(value) end box_types["string"] = StringBox.new +local IPv4Box = {} +function IPv4Box.new(address) + local ret = {root={}} + if address then ret.box = ipv4:pton(address) end + return setmetatable(ret, {__index=IPv4Box}) +end +function IPv4Box:get() + return self.box +end +function IPv4Box:set(address) + self.box = assert(ipv4:pton(address)) +end +box_types["inet:ipv4-address"] = IPv4Box.new + +local IPv6Box = {} +function IPv6Box.new(address) + local ret = {} + if address then ret.box = ipv6:pton(address) end + return setmetatable(ret, {__index=IPv6Box}) +end +function IPv6Box:get() + return self.box +end +function IPv6Box:set(address) + self.box = assert(ipv6:pton(address)) +end +box_types["inet:ipv6-address"] = IPv6Box.new + local IPv4PrefixBox = {} function IPv4PrefixBox.new() local ret = {root={}} @@ -163,7 +190,6 @@ box_types["enumeration"] = Enum.new local Union = {} function Union.new(types) local ret = {types={}} - for _, name in pairs(types) do -- 9.12 specifies unions cannot contain "leafref" or "empty" if name == "empty" or name == "leafref" then @@ -171,16 +197,17 @@ function Union.new(types) end ret.types[name] = create_box(name) end - return setmetatable(ret, {__index=Union}) end function Union:get() - return self.value + if self.box then + return self.box:get() + end end function Union:set(v) - local function setbox(dest, val) dest.value = val end + local function setbox(b, v) b:set(v) end for _, box in pairs(self.types) do local valid = pcall(setbox, box, v) if valid then @@ -190,6 +217,7 @@ function Union:set(v) end error(("Unable to find matching type for '%s' (%s)"):format(v, type(v))) end + box_types["union"] = Union.new Container = {} @@ -337,7 +365,7 @@ function selftest() con.testbox = "8.8.8.8" assert(ipv4:ntop(con.testbox) == "8.8.8.8") asserterror(setvalue, con, "testbox", "should fail") - + -- Test the IPv4 box (both valid and invalid data). root.testbox = box_types["inet:ipv4-address"]() con.testbox = "8.8.8.8" diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index b0f0ef55b8..b162c14ade 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -282,8 +282,6 @@ function parse_file(filename) end function selftest() - local S = require("syscall") - local function assert_equal(a, b) if not lib.equal(a, b) then print(a, b) @@ -331,11 +329,6 @@ function selftest() return table.concat({...}, "\n") end - local function file_exists(path) - local stat = S.stat(path) - return stat and stat.isreg - end - -- Test the string parser test_string("foo", "foo") test_string([["foo"]], "foo") @@ -362,11 +355,12 @@ function selftest() test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type"}}}}) - local filename = "lib/yang/example.yang" - local pwd = S.getcwd() - -- If current path is root append src/ to filename - if file_exists(pwd.."/src/snabb") then - filename = "src/"..filename - end - parse_file(filename) + -- We need to locate the yang example code in a reliable way, we can't + -- give the path relative to the executable as the current working + -- directly could be different. To do this we find the absolute path to + -- the snabb executable and then build the path to the yang file from that. + local S = require("syscall") + local snabb_exe = S.readlink("/proc/self/exe"):gsub("(.-)[%w_-]*$", "%1") + local yang_example = snabb_exe.."lib/yang/example.yang" + parse_file(yang_example) end From a85017b9acc739f8c196565d61b16750671534e0 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 19 Sep 2016 11:47:50 +0000 Subject: [PATCH 022/631] engine: Extend API for spawning a background worker. New functions: start(options) stop() status() -> 'unstarted' | 'running' | 'stopped' --- src/core/app.lua | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/core/app.lua b/src/core/app.lua index 7a6482fbee..d217ed24b7 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -14,6 +14,7 @@ local zone = require("jit.zone") local jit = require("jit") local ffi = require("ffi") local C = ffi.C +local S = require("syscall") require("core.packet_h") -- Packet per pull @@ -252,6 +253,42 @@ function apply_config_actions (actions, conf) end end +-- Child process support + +local child + +-- Start an asynchronous child process to execute the engine configuration. +-- Options are passed to engine.main() in the child. +function start (options) + local pid = S.fork() + if pid == 0 then + -- This is the child worker process + main(options) + print("terminated") + os.exit(0) + else + -- This is the parent supervisor process + child = pid + print("child", child) + end +end + +-- Stop the child process. +function stop () + if child then S.kill(child, "term") end +end + +-- Return status of child process: 'unstarted' | 'running' | 'stopped'. +function status () + if child == nil then + return 'unstarted' + elseif S.waitid("pid", child, "exited, nohang") then + return 'running' + else + return 'stopped' + end +end + -- Call this to "run snabb switch". function main (options) options = options or {} From ecee1a5efe9090c8403eb41e0db6f83e88b03dff Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 30 Aug 2016 11:20:39 +0200 Subject: [PATCH 023/631] Initial commit of snabb's lwAFTR YANG config --- src/apps/lwaftr/snabb-softwire.yang | 304 ++++++++++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 src/apps/lwaftr/snabb-softwire.yang diff --git a/src/apps/lwaftr/snabb-softwire.yang b/src/apps/lwaftr/snabb-softwire.yang new file mode 100644 index 0000000000..82abd6c912 --- /dev/null +++ b/src/apps/lwaftr/snabb-softwire.yang @@ -0,0 +1,304 @@ +module snabb-softwire { + namespace "snabb:lwaftr"; + prefix "softwire"; + + import ietf-inet-types {prefix inet; } + + organization "Igalia"; + contact "Jessica Tallon "; + description ""; + + revision 2016-08-01 { + description "Initial revision."; + reference "-00"; + } + + typedef PacketPolicy { + type enumeration { + enum allow { + value 0; + description "Allow pakcets."; + } + enum deny { + value 1; + description "Deny packets."; + } + } + description "Policy for what to do with packets, either allow or deny + them."; + } + + typedef PositiveNumber { + type uint32 { + range "1.. 2147483647"; + } + description "This is a non-negative integer value which is 1 or above."; + } + + typedef VlanTag { + type uint16 { + range "0..4095"; + } + description "This is a VLAN tag according to 802.1Q Ethernet tagging."; + } + + container softwire-config { + description "Configuration for Snabb lwaftr."; + + leaf ipv4-ip { + type inet:ipv4-address; + mandatory true; + description "Sets the IPv4 address of the internet facing NIC."; + } + + leaf ipv6-ip { + type inet:ipv6-address; + mandatory true; + description "Sets the IPv6 address of the internal facing NIC."; + } + + leaf mac-b4 { + type inet:mac-address; + mandatory true; + description "Sets the MAC address of the internal facing NIC."; + } + + leaf mac-inet { + type inet:mac-address; + mandatory true; + description "Sets the MAC address of the internet facing NIC."; + } + + leaf hairpinning { + type boolean; + default true; + description "Enables or disables hairpinning that is a requirement of + RFC 7596."; + } + + + container mtu { + description "The MTU settings are used to determine whether a packet + needs to be fragmented. The MTU handling is otherwise + underdeveloped. It is not dynamically updated upon receiving + ICMP packets too big."; + + leaf ipv4-mtu { + type uint16; + default 1460; + description "Sets the MTU value for the IPv4 NIC."; + } + + leaf ipv6-mtu { + type uint16; + default 1500; + description "Set the MTU value for the IPv6 NIC."; + } + } + + container L2-next-hop { + description "One might expect to set the default gateway IPv4 and IPv6 + addresses and have those resolved via ARP, however ARP is + not yet supported via the lwAFTR. One must specify either + the MAC Address of the next hop host however you could + instead specify the IP addresses of the next hops and have + the lwAFTR resolve the corresponding MAC addresses via NDP + and ARP."; + + leaf b4-mac { + type inet:mac-address; + description "Sets the MAC address of the next hop for the internal + facing NIC."; + } + + leaf inet-mac { + type inet:mac-address; + description "Sets the MAC address of the next hop for the internet + facing NIC."; + } + + leaf inet-ipv4 { + type inet:ipv4-address; + description "Sets the IPv4 address of the next hop for the internet + facing NIC that will be used to resolve the MAC address."; + } + + leaf b4-ipv6 { + type inet:ipv6-address; + description "Sets the IPv4 address of the next hop for the internal + facing NIC that will be used to resolve the MAC address."; + } + } + + + container rate-limiting-icmp { + description "ICMP rate limiting is mandated by many RFCs. This configures + the number of ICMP packets the lwAFTR can send for the given + time period. Lower values are recomended for + non-experimental use."; + + leaf packets { + type uint32; + default 600000; + description "The number of packets which can be sent within the time + period."; + } + + leaf icmpv6-rate-limiter-n-seconds { + type PositiveNumber; + default 2; + description "The time period given in seconds."; + } + } + + container icmp-handling-policies { + description "The lwAFTR can be configured to allow or drop incoming and + outgoing ICMPv4 and ICMPv6 messages."; + + leaf icmpv4-incoming { + type PacketPolicy; + default allow; + description "Sets the policy for incoming ICMPv4 messages."; + } + + leaf icmpv4-outgoing { + type PacketPolicy; + default allow; + description "Sets the policy for outgoing ICMPv4 messages."; + } + + leaf icmpv6-incoming { + type PacketPolicy; + default allow; + description "Sets the policy for incoming ICMPv6 messages."; + } + + leaf icmpv6-outgoing { + type PacketPolicy; + default allow; + description "Sets the policy for outgoing ICMPv6 messages."; + } + } + + container vlan-tagging { + description "This configures 802.1Q Ethernet tagging. If tagging is + enabled the outbound and inbound tags must be specified."; + + leaf vlan-tagging { + type boolean; + default true; + description "Enables or disables VLAN tagging."; + } + + leaf v4-tag { + type VlanTag; + description "Sets the tag for v4"; + } + + leaf v6-vlan-tag { + type VlanTag; + description "Sets the tag for v6"; + } + } + + container filters { + description "This configures the ingress and egress filters for each NIC. + If set, these should be a pflang filter, pflang is the + language of tcpdump, libpcap and other tools. The filters + will run on the packets without vlan tags, if any."; + + leaf ipv4-ingress-filter { + type string; + description "Sets the pflang filter for IPv4 ingress."; + } + + leaf ipv4-egress-filter { + type string; + description "Sets the pflang filter for IPv4 egress."; + } + + leaf ipv6-ingress-filter { + type string; + description "Sets the pflang filter for IPv6 ingress."; + } + + leaf ipv6-egress-filter { + type string; + description "Sets the pflang filter for IPv6 egress."; + } + } + } + + container binding-table { + description "The binding table is a collection of softwire tunnels. One + endpoint of the softwire is the lwAFTR and the other is the + B4. It contains information so the lwAFTR knows which IPv4 + (or part of an IPv4 address) is associated with which B4."; + + list psid_map { + key "ipv4-address"; + description "Defines the set of IPv4 addresses that are provisioned by a + lwAFTR. It also defines the way in which those addresses are + shared by specifying the psid_length and shift. See RFC 7597 + for more details on the PSID scheme for sharing IPv4 + addresses."; + + leaf ipv4-address { + type inet:ipv4-address; + mandatory true; + description "The IPv4 address that is provisioned"; + } + + leaf psid_length { + type uint16; + default 0; + } + + leaf shift { + type uint16; + default 16; + } + } + + list br-addresses { + key "ipv6-address"; + + leaf ipv6-address { + type inet:ipv6-address; + } + } + + list binding-table-entry { + key "ipv4-address psid"; + + leaf ipv4-address { + type inet:ipv4-address; + mandatory true; + description "Public IPv4 address of the softwire."; + } + + leaf psid { + type uint16; + mandatory true; + description "Port set ID"; + } + + leaf padding { + type uint16; + description "Zeros"; + } + + leaf br { + type uint32; + mandatory true; + description "Border router"; + } + + leaf ipv6-address { + type inet:ipv6-address; + mandatory true; + description "Address of the B4"; + } + } + } +} From a7c219423b25a5c444c36c52a784413eef8897bc Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Thu, 29 Sep 2016 15:16:29 +0000 Subject: [PATCH 024/631] core.shm: Added shm.alias(newpath, oldpath) Added an API for creating alias names for shm objects. The aliases are implemented as symbolic links on the filesystem. --- src/README.md | 4 ++++ src/core/shm.lua | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/README.md b/src/README.md index 6dcbb3955e..6b72faec2b 100644 --- a/src/README.md +++ b/src/README.md @@ -483,6 +483,10 @@ If *readonly* is non-nil the shared object is mapped in read-only mode. *Readonly* defaults to nil. Fails if the shared object does not already exist. Returns a pointer to the mapped object. +— Function **shm.alias** *new-path* *existing-path* + +Create an alias (symbolic link) for an object. + — Function **shm.exists** *name* Returns a true value if shared object by *name* exists. diff --git a/src/core/shm.lua b/src/core/shm.lua index 15b2f3f440..3b97847731 100644 --- a/src/core/shm.lua +++ b/src/core/shm.lua @@ -61,6 +61,11 @@ function exists (name) return fd and fd:close() end +function alias (name, target) + assert(S.symlink(root.."/"..resolve(target), root.."/"..resolve(name)), + "shm alias failed") +end + function resolve (name) local q, p = name:match("^(/*)(.*)") -- split qualifier (/) local result = p From edee5b6c2470bf01a64f8554bcc24d3d103831c7 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Thu, 29 Sep 2016 15:17:19 +0000 Subject: [PATCH 025/631] program/worker: Added "worker" program This is a work in progress. The new command is: snabb worker NAME PARENTPID CORE and the effect is to spawn a new Snabb process that simply executes an app network and reacts to configuration changes. The worker process looks in a "group/" shm folder of the PARENTPID to keep track of its current configuration. This is intended as an internal interface that the engine can use to parallelize an app network by delegating parts to different child processes. The processes would be managed directly by the engine and end users need not be aware of the "snabb worker" interface. --- src/program/worker/README | 10 ++++++ src/program/worker/README.inc | 1 + src/program/worker/worker.lua | 66 +++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 src/program/worker/README create mode 120000 src/program/worker/README.inc create mode 100644 src/program/worker/worker.lua diff --git a/src/program/worker/README b/src/program/worker/README new file mode 100644 index 0000000000..dd2ebae8b5 --- /dev/null +++ b/src/program/worker/README @@ -0,0 +1,10 @@ +Usage: worker [OPTION]... NAME PARENTPID CORE + +Worker: Execute a Snabb app network and process configuration updates. + +Configuration NAME is loaded from process PARENTPID and executed. The +configuration is reloaded any time the parent makes an update. + +The worker process executes with affinity to one CPU core (CORE). + + -h, --help Print this usage message. diff --git a/src/program/worker/README.inc b/src/program/worker/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/worker/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/worker/worker.lua b/src/program/worker/worker.lua new file mode 100644 index 0000000000..92f1167ffa --- /dev/null +++ b/src/program/worker/worker.lua @@ -0,0 +1,66 @@ +-- worker.lua - Snabb engine worker process to execute an app network +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(..., package.seeall) + +local lib = require("core.lib") +local usage = require("program.worker.README_inc") +local shm = require("core.shm") +local ffi = require("ffi") +local C = ffi.C +local S = require("syscall") + +function run (parameters) + if #parameters ~= 3 then + print(usage) + os.exit(1) + end + -- Parse a numeric parameter + local function num (string) + if string:match("^[0-9]+$") then + return tonumber(string) + else + print("bad number: " .. string) + main.exit(1) + end + end + local name = parameters[1] + local parent = num(parameters[2]) + local core = num(parameters[3]) + print(("Starting worker %s on core %d for parent %d"):format(name, core, parent)) + + -- Setup affinity + if core then S.sched_setaffinity(0, {core}) end + + -- Create "group" alias to the shared group folder in the parent process + shm.alias("group", "/"..parent.."/group") + + -- Wait for parent to provide an initial configuration + local warned + while not shm.exists("group/"..name.."/configs.counter") do + if not warned then + print("waiting for configuration...") + warned = true + C.usleep(1000) + end + end + + -- Map the counter for how many times our configuration has been updated. + -- This provides an efficient way to poll for updates. + local configs = shm.map("group/"..name.."/configs", "counter") + + -- Run the engine with continuous configuration updates + local current_config + local update = function () return current_config ~= counter.read(configs) end + while true do + if update() then + -- note: read counter _before_ config file to avoid a race + current_config = counter.read(configs) + local c = config.readfile(shm.path("group/config/"..name)) + engine.configure(c) + end + -- Run until next update + engine.main({done = update, no_report = true}) + end +end + From 42f25ee125c7f79475b631cb333a3896faf12066 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 30 Sep 2016 09:37:18 +0000 Subject: [PATCH 026/631] program.worker: Removed (for replacement) Having a complete program for the worker process is overkill. This will be replaced by a core.worker module. --- src/program/worker/README | 10 ------ src/program/worker/README.inc | 1 - src/program/worker/worker.lua | 66 ----------------------------------- 3 files changed, 77 deletions(-) delete mode 100644 src/program/worker/README delete mode 120000 src/program/worker/README.inc delete mode 100644 src/program/worker/worker.lua diff --git a/src/program/worker/README b/src/program/worker/README deleted file mode 100644 index dd2ebae8b5..0000000000 --- a/src/program/worker/README +++ /dev/null @@ -1,10 +0,0 @@ -Usage: worker [OPTION]... NAME PARENTPID CORE - -Worker: Execute a Snabb app network and process configuration updates. - -Configuration NAME is loaded from process PARENTPID and executed. The -configuration is reloaded any time the parent makes an update. - -The worker process executes with affinity to one CPU core (CORE). - - -h, --help Print this usage message. diff --git a/src/program/worker/README.inc b/src/program/worker/README.inc deleted file mode 120000 index 100b93820a..0000000000 --- a/src/program/worker/README.inc +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/program/worker/worker.lua b/src/program/worker/worker.lua deleted file mode 100644 index 92f1167ffa..0000000000 --- a/src/program/worker/worker.lua +++ /dev/null @@ -1,66 +0,0 @@ --- worker.lua - Snabb engine worker process to execute an app network --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - -module(..., package.seeall) - -local lib = require("core.lib") -local usage = require("program.worker.README_inc") -local shm = require("core.shm") -local ffi = require("ffi") -local C = ffi.C -local S = require("syscall") - -function run (parameters) - if #parameters ~= 3 then - print(usage) - os.exit(1) - end - -- Parse a numeric parameter - local function num (string) - if string:match("^[0-9]+$") then - return tonumber(string) - else - print("bad number: " .. string) - main.exit(1) - end - end - local name = parameters[1] - local parent = num(parameters[2]) - local core = num(parameters[3]) - print(("Starting worker %s on core %d for parent %d"):format(name, core, parent)) - - -- Setup affinity - if core then S.sched_setaffinity(0, {core}) end - - -- Create "group" alias to the shared group folder in the parent process - shm.alias("group", "/"..parent.."/group") - - -- Wait for parent to provide an initial configuration - local warned - while not shm.exists("group/"..name.."/configs.counter") do - if not warned then - print("waiting for configuration...") - warned = true - C.usleep(1000) - end - end - - -- Map the counter for how many times our configuration has been updated. - -- This provides an efficient way to poll for updates. - local configs = shm.map("group/"..name.."/configs", "counter") - - -- Run the engine with continuous configuration updates - local current_config - local update = function () return current_config ~= counter.read(configs) end - while true do - if update() then - -- note: read counter _before_ config file to avoid a race - current_config = counter.read(configs) - local c = config.readfile(shm.path("group/config/"..name)) - engine.configure(c) - end - -- Run until next update - engine.main({done = update, no_report = true}) - end -end - From 4922af892127ecb667837b11596026e236c4c017 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 30 Sep 2016 09:38:25 +0000 Subject: [PATCH 027/631] core.worker: New module to run worker processes This module adds a simple API for spawning "worker" child processes that each run an app network on a dedicated core. The app network is provided and updated by the parent process. --- src/core/worker.lua | 110 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/core/worker.lua diff --git a/src/core/worker.lua b/src/core/worker.lua new file mode 100644 index 0000000000..2a5a0d1543 --- /dev/null +++ b/src/core/worker.lua @@ -0,0 +1,110 @@ +-- worker.lua - Execute "worker" child processes to execute app networks +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +-- API: +-- start(name, core) +-- stop(name) +-- status() -> table of { name = } +-- configure(name, config) + +-------------------------------------------------------------- +-- Master (parent) process code +-------------------------------------------------------------- + +local children = {} + +local function child (name) + return children[name] or error("no such child: " .. name) +end + +-- Start a worker process with affinity to a specific CPU core. +-- The child will execute an app network when provided with configure(). +function start (name, core) + local pid = S.fork() + if pid ~= 0 then + -- Child (worker) process + init(name, core, S.getppid()) + else + -- Parent process + children[name] = { pid = pid, core = core } + end +end + +-- Terminate a child process +function stop (name) + S.kill(child(name), 'kill') +end + +-- Return information about all worker processes in a table. +function status () + local status = {} + for name, info in pairs(children) do + status[name] = { + pid = info.pid, + alive = (S.kill(info.pid, 0) == 0) + } + end + return status +end + +-- Configure a worker process with a new app network. +function configure (name, c) + -- Ensure "configs" shm counter exists for child to poll + local child = children[name] + local child_path = "group/child/"..name + if not child.configs then + child.configs = shm.map(child_path.."/configs", {"counter"}, false, true) + end + config.save(shm.path(child_path.."/config"), c) + counter.add(child.configs, 1) +end + +-------------------------------------------------------------- +-- Worker (child) process code +-------------------------------------------------------------- + +-- Initialize the worker by attaching to relevant shared memory +-- objects and entering the main engine loop. +function init (name, core, parentpid) + local name = parameters[1] + local parent = num(parameters[2]) + local core = num(parameters[3]) + print(("Starting worker %s on core %d for parent %d"):format(name, core, parent)) + + -- Setup affinity + if core then S.sched_setaffinity(0, {core}) end + + -- Create "group" alias to the shared group folder in the parent process + shm.alias("group", "/"..parent.."/group") + + -- Wait for parent to provide an initial configuration + local warned + while not shm.exists("group/"..name.."/configs.counter") do + if not warned then + print("waiting for configuration...") + warned = true + C.usleep(1000) + end + end + + -- Map the counter for how many times our configuration has been updated. + -- This provides an efficient way to poll for updates. + local configs = shm.map("group/"..name.."/configs", "counter") + + -- Run the engine with continuous configuration updates + local current_config + local child_path = "group/config/..name" + local update = function () return current_config ~= counter.read(configs) end + while true do + if update() then + -- note: read counter _before_ config file to avoid a race + current_config = counter.read(configs) + local c = config.load(shm.path(child_path.."/config")) + engine.configure(c) + end + -- Run until next update + engine.main({done = update, no_report = true}) + end +end + + From 0d717a46e924e5b77e0e751ab4aee9a88b435b3f Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 30 Sep 2016 10:07:06 +0000 Subject: [PATCH 028/631] Revert "engine: Extend API for spawning a background worker." Obsoleted by core.worker module. This reverts commit a85017b9acc739f8c196565d61b16750671534e0. --- src/core/app.lua | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index d217ed24b7..7a6482fbee 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -14,7 +14,6 @@ local zone = require("jit.zone") local jit = require("jit") local ffi = require("ffi") local C = ffi.C -local S = require("syscall") require("core.packet_h") -- Packet per pull @@ -253,42 +252,6 @@ function apply_config_actions (actions, conf) end end --- Child process support - -local child - --- Start an asynchronous child process to execute the engine configuration. --- Options are passed to engine.main() in the child. -function start (options) - local pid = S.fork() - if pid == 0 then - -- This is the child worker process - main(options) - print("terminated") - os.exit(0) - else - -- This is the parent supervisor process - child = pid - print("child", child) - end -end - --- Stop the child process. -function stop () - if child then S.kill(child, "term") end -end - --- Return status of child process: 'unstarted' | 'running' | 'stopped'. -function status () - if child == nil then - return 'unstarted' - elseif S.waitid("pid", child, "exited, nohang") then - return 'running' - else - return 'stopped' - end -end - -- Call this to "run snabb switch". function main (options) options = options or {} From 6e28390e43ca8878af72cd8ca7e85d49fac8bb1a Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Sun, 2 Oct 2016 17:22:49 +0000 Subject: [PATCH 029/631] shm: mkdir() now creates full path Previously there was an implicit dirname() that skipped the last path component. Removed this restriction to make the subroutine more useful. --- src/core/shm.lua | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/shm.lua b/src/core/shm.lua index 3b97847731..c690972568 100644 --- a/src/core/shm.lua +++ b/src/core/shm.lua @@ -29,7 +29,7 @@ local function map (name, type, readonly, create) local fd, err if create then -- Create the parent directories. If this fails then so will the open(). - mkdir(path) + mkdir(lib.dirname(path)) fd, err = S.open(root..'/'..path, "creat, rdwr", "rwxu") else fd, err = S.open(root..'/'..path, readonly and "rdonly" or "rdwr") @@ -73,8 +73,7 @@ function resolve (name) return result end --- Make directories needed for a named object. --- Given the name "foo/bar/baz" create /var/run/foo and /var/run/foo/bar. +-- Make the named subdirectory in the shm folder. function mkdir (name) -- Create root with mode "rwxrwxrwt" (R/W for all and sticky) if it -- does not exist yet. @@ -88,7 +87,10 @@ function mkdir (name) -- Create sub directories local dir = root name:gsub("([^/]+)", - function (x) S.mkdir(dir, "rwxu") dir = dir.."/"..x end) + function (x) + dir = dir.."/"..x + S.mkdir(dir, "rwxu") + end) end -- Delete a shared object memory mapping. From f614c6a6bf0f902a9c911bf87d7a863bd72fa09a Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Sun, 2 Oct 2016 17:23:50 +0000 Subject: [PATCH 030/631] core.memory: Manage hugetlb file-backing lifetime HugeTLB memory allocated for DMA is now kept accessible in file-backed shared memory. This makes it possible for process to map each others' memory. The new function shutdown(pid) removes file-backing for a process that is terminating (in order to free memory for the OS). More specifically: Allocating a hugetlb page with address xxxxx creates: /var/run/snabb/hugetlb/xxxxx.dma File backed shared memory /var/run/snabb/$pid/group/dma/xxxxx.dma Symlink to above and the shutdown(pid) function will follow the symlinks to remove the file backing. (The reason for symlinks is that these directories are on separate filesystems, one hugetlbfs and one tmpfs.) --- src/core/memory.lua | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/core/memory.lua b/src/core/memory.lua index 5c771bd2c0..08d206d2f3 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -11,6 +11,7 @@ module(...,package.seeall) local ffi = require("ffi") local C = ffi.C local syscall = require("syscall") +local shm = require("core.shm") local lib = require("core.lib") @@ -60,7 +61,7 @@ function allocate_hugetlb_chunk () assert(fd, tostring(err)) fd:flock("ex") for i =1, 3 do - local page = allocate_huge_page(huge_page_size) + local page = allocate_huge_page(huge_page_size, true) if page ~= nil then fd:flock("un") fd:close() @@ -152,13 +153,20 @@ function allocate_huge_page (size, persistent) local phys = resolve_physical(tmpptr) local virt = bit.bor(phys, tag) local ptr = syscall.mmap(virt, size, "read, write", "shared, hugetlb, fixed", fd, 0) + local filename = ("/var/run/snabb/hugetlbfs/%012x.dma"):format(tonumber(phys)) if persistent then - assert(syscall.rename(tmpfile, "/var/run/snabb/hugetlbfs/dma.%012x", phys)) + assert(syscall.rename(tmpfile, filename)) + shm.mkdir(shm.resolve("group/dma")) + syscall.symlink(filename, shm.root..'/'..shm.resolve("group/dma/"..lib.basename(filename))) else assert(syscall.unlink(tmpfile)) end syscall.close(fd) - return ptr + return ptr, filename +end + +function hugetlb_filename (address) + return ("%012x.dma"):format(virtual_to_physical(address)) end -- resolve_physical(ptr) => uint64_t @@ -186,6 +194,20 @@ function ensure_hugetlbfs () end end +-- Deallocate all file-backed shared memory allocated by pid (or other +-- processes in its process group). +-- +-- This is an internal API function provided for cleanup during +-- process termination. +function shutdown (pid) + local dma = shm.children("/"..pid.."/group/dma") + for _, file in ipairs(dma) do + local symlink = shm.root.."/"..pid.."/group/dma/"..file + local realfile = syscall.readlink(symlink) + syscall.unlink(realfile) + end +end + --- ### selftest function selftest (options) From 50a446bef7341f8f6e1a1c375cc551234fa9e1b4 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Sun, 2 Oct 2016 17:27:46 +0000 Subject: [PATCH 031/631] hardware.pci: Disable bus mastering on shutdown The new function shutdown(pid) will disable PCI bus mastering (i.e. DMA) for all PCI devices that were in use by the process (and its process group). This is intended as a safety mechanism to prevent "dangling" DMA requests on a device after a Snabb process terminates and the DMA memory is returned to the kernel. To keep track of which devices are in use we now create shm files with names like "group/dma/pci/01:00.0" for each device that we have enabled bus mastering for. These are the devices that are reverted by shutdown(). --- src/lib/hardware/pci.lua | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/lib/hardware/pci.lua b/src/lib/hardware/pci.lua index de34a4d5d7..2c908f2ac4 100644 --- a/src/lib/hardware/pci.lua +++ b/src/lib/hardware/pci.lua @@ -5,6 +5,7 @@ module(...,package.seeall) local ffi = require("ffi") local C = ffi.C local S = require("syscall") +local shm = require("core.shm") local lib = require("core.lib") @@ -148,6 +149,7 @@ function map_pci_memory (device, n, lock) assert(mem, tostring(err)) return ffi.cast("uint32_t *", mem), f end + function close_pci_resource (fd, base) local st, err = fd:stat() assert(st, tostring(err)) @@ -166,14 +168,28 @@ function set_bus_master (device, enable) local value = ffi.new("uint16_t[1]") assert(C.pread(fd, value, 2, 0x4) == 2) if enable then + shm.create('group/dma/pci/'..canonical(device), 'uint64_t') value[0] = bit.bor(value[0], lib.bits({Master=2})) else + shm.unlink('group/dma/pci/'..canonical(device)) value[0] = bit.band(value[0], bit.bnot(lib.bits({Master=2}))) end assert(C.pwrite(fd, value, 2, 0x4) == 2) f:close() end +-- Shutdown DMA to prevent "dangling" requests for PCI devices opened +-- by pid (or other processes in its process group). +-- +-- This is an internal API function provided for cleanup during +-- process termination. +function shutdown (pid) + local dma = shm.children("/"..pid.."/group/dma/pci") + for _, device in ipairs(dma) do + set_bus_master(device, false) + end +end + function root_check () lib.root_check("error: must run as root to access PCI devices") end From 9fe91ac51483232372abfc042e5120c5ad7d4be1 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Sun, 2 Oct 2016 17:30:22 +0000 Subject: [PATCH 032/631] main: Call pci.shutdown() and memory.shutdown() Call pci.shutdown() and memory.shutdown() when terminating the main Snabb process (not its worker children). The pci.shutdown() call will disabled DMA for all PCI devices that were in use. The memory.shutdown() will then delete the DMA memory backing files so that the memory will be returned to the kernel (after all processes that have mapped the memory terminate). --- src/core/main.lua | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/core/main.lua b/src/core/main.lua index 5930142313..585d662eaf 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -134,6 +134,21 @@ end -- Cleanup after Snabb process. function shutdown (pid) + -- Parent process performs additional cleanup steps. + -- (Parent is the process whose 'group' folder is not a symlink.) + local st, err = S.lstat(shm.root.."/"..pid.."/group") + local is_parent = st and st.isdir + if is_parent then + -- simple pcall helper to print error and continue + local function safely (f) + local ok, err = pcall(f) + if not ok then print(err) end + end + -- Run cleanup hooks + safely(function () require("lib.hardware.pci").shutdown(pid) end) + safely(function () require("core.memory").shutdown(pid) end) + end + -- Free shared memory objects if not _G.developer_debug and not lib.getenv("SNABB_SHM_KEEP") then shm.unlink("/"..pid) end From 7fbc1a3af91f63f14469a57372b2a6bc34a8571f Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Mon, 3 Oct 2016 12:51:03 +0000 Subject: [PATCH 033/631] core.memory: Auto-mapping of DMA memory w/ SIGSEGV Added a SIGSEGV handler that attempts to automatically map addresses corresponding to DMA memory. The mapping succeeds when the DMA memory was allocated by any Snabb process in the process group. If the mapping does not succeed then the standard behavior of SIGSEGV is triggered. Includes a simple unit test in core.memory to allocate DMA memory, unmap it, and then access it anyway. Confirms that the signal handler runs the expected number of times. --- src/core/memory.c | 100 ++++++++++++++++++++++++++++++++++++++++++++ src/core/memory.h | 2 + src/core/memory.lua | 25 ++++++++++- 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 src/core/memory.c create mode 100644 src/core/memory.h diff --git a/src/core/memory.c b/src/core/memory.c new file mode 100644 index 0000000000..4f6cfc3f5c --- /dev/null +++ b/src/core/memory.c @@ -0,0 +1,100 @@ +// memory.c - supporting code for DMA memory management +// Use of this source code is governed by the Apache 2.0 license; see COPYING. + +// This file implements a SIGSEGV handler that traps access to DMA +// memory that is available to the Snabb process but not yet mapped. +// These accesses are transparently supported by mapping the required +// memory "just in time." +// +// The overall effect is that each Snabb process in a group can +// transparently access the DMA memory allocated by other processes. +// DMA pointers in each process are also valid in all other processes +// within the same group. +// +// Note that every process maps DMA memory to the same address i.e. +// the physical address of the memory with some tag bits added. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// See memory.lua for pointer tagging scheme. +#define TAG 0x500000000000ULL +#define PATH_MAX 256 + +static uint64_t page_mask; +static char path_template[PATH_MAX]; + +// Counter for the number of times a page has been mapped on-demand by +// the SIGSEGV handler. +int memory_demand_mappings; + +static void memory_sigsegv_handler(int sig, siginfo_t *si, void *unused); + +// Install signal handler +static void set_sigsegv_handler() +{ + struct sigaction sa; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + sa.sa_sigaction = memory_sigsegv_handler; + assert(sigaction(SIGSEGV, &sa, NULL) != -1); +} + +static void memory_sigsegv_handler(int sig, siginfo_t *si, void *unused) +{ + int fd = -1; + struct stat st; + char path[PATH_MAX]; + uint64_t address = (uint64_t)si->si_addr; + uint64_t page = address & ~TAG & page_mask; + // Disable this handler to avoid potential recursive signals. + signal(SIGSEGV, SIG_DFL); + fflush(stdout); + // Check that this is a DMA memory address + if ((address & TAG) != TAG) { + goto punt; + } + snprintf(path, PATH_MAX, path_template, page); + // Check that the memory is accessible to this process + if ((fd = open(path, O_RDWR)) == -1) { + goto punt; + } + if (fstat(fd, &st) == -1) { + goto punt; + } + // Map the memory at the expected address + if (mmap((void *)(page | TAG), st.st_size, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_FIXED|MAP_HUGETLB, fd, 0) == MAP_FAILED) { + goto punt; + } + close(fd); + memory_demand_mappings++; + // Re-enable the handler for next time + set_sigsegv_handler(); + return; + punt: + // Fall back to the default SEGV behavior by resending the signal + // now that the handler is disabled. + // See https://www.cons.org/cracauer/sigint.html + kill(getpid(), SIGSEGV); +} + +// Setup a SIGSEGV handler to map DMA memory on demand. +void memory_sigsegv_setup(int huge_page_size, const char *path) +{ + // Save parameters + page_mask = ~(uint64_t)(huge_page_size - 1); + assert(strlen(path) < PATH_MAX); + strncpy(path_template, path, PATH_MAX); + memory_demand_mappings = 0; + set_sigsegv_handler(); +} + diff --git a/src/core/memory.h b/src/core/memory.h new file mode 100644 index 0000000000..6dab2476c5 --- /dev/null +++ b/src/core/memory.h @@ -0,0 +1,2 @@ +int memory_demand_mappings; +void memory_sigsegv_setup(int huge_page_size, const char *path); diff --git a/src/core/memory.lua b/src/core/memory.lua index 08d206d2f3..d62a407d23 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -12,8 +12,8 @@ local ffi = require("ffi") local C = ffi.C local syscall = require("syscall") local shm = require("core.shm") - local lib = require("core.lib") +require("core.memory_h") --- ### Serve small allocations from hugepage "chunks" @@ -208,12 +208,17 @@ function shutdown (pid) end end +-- Setup SIGSEGV handler to automatically map memory from other processes +C.memory_sigsegv_setup(huge_page_size, + shm.root..'/'..shm.resolve("group/dma/%012lx.dma")) + --- ### selftest function selftest (options) print("selftest: memory") print("Kernel vm.nr_hugepages: " .. syscall.sysctl("vm.nr_hugepages")) ensure_hugetlbfs() -- can print a message, let that go first + local dmapointers = {} for i = 1, 4 do io.write(" Allocating a "..(huge_page_size/1024/1024).."MB HugeTLB:") io.flush() @@ -223,8 +228,26 @@ function selftest (options) print(" Virtual address: 0x" .. bit.tohex(ffi.cast(uint64_t, dmaptr), 12)) ffi.cast("uint32_t*", dmaptr)[0] = 0xdeadbeef -- try a write assert(dmaptr ~= nil and dmalen == huge_page_size) + table.insert(dmapointers, dmaptr) end print("Kernel vm.nr_hugepages: " .. syscall.sysctl("vm.nr_hugepages")) + print("Testing automatic remapping of DMA memory") + local orig_demand_mappings = C.memory_demand_mappings + -- First unmap all of the DMA memory + for _, dmaptr in ipairs(dmapointers) do + print(" Unmapping " .. tostring(dmaptr)) + assert(syscall.munmap(dmaptr, huge_page_size)) + end + -- Now touch them all + for _, dmaptr in ipairs(dmapointers) do + print(" Writing ".. tostring(dmaptr)) + dmaptr[0] = 42 + end + local new_demand_mappings = C.memory_demand_mappings - orig_demand_mappings + print(("Created %d on-demand memory mappings with SIGSEGV handler."):format( + new_demand_mappings)) + assert(new_demand_mappings >= #dmapointers) + -- Now access it and rely on the SIGSEGV handler to print("HugeTLB page allocation OK.") end From f4228ca7396e00f220a73863cd64bb8750ec6712 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 4 Oct 2016 10:11:31 +0000 Subject: [PATCH 034/631] main: Add SNABB_PROGRAM_LUACODE env var The new environment variable SNABB_PROGRAM_LUACODE can be used to force a new Snabb process to execute a string of Lua code instead of running the standard program-selection logic. This is intended to support Snabb processes that want to spin off helper processes that need to execute specific code and be independent of the usual command-line syntax (e.g. sensitivity to argv[0] etc). --- src/core/main.lua | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index 585d662eaf..ede9bde0f1 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -41,12 +41,19 @@ function main () error("fatal: "..ffi.os.."/"..ffi.arch.." is not a supported platform\n") end initialize() - local program, args = select_program(parse_command_line()) - if not lib.have_module(modulename(program)) then - print("unsupported program: "..program:gsub("_", "-")) - usage(1) + if lib.getenv("SNABB_PROGRAM_LUACODE") then + -- Run the given Lua code instead of the command-line + local expr = lib.getenv("SNABB_PROGRAM_LUACODE") + loadstring(expr)() else - require(modulename(program)).run(args) + -- Choose a program based on the command line + local program, args = select_program(parse_command_line()) + if not lib.have_module(modulename(program)) then + print("unsupported program: "..program:gsub("_", "-")) + usage(1) + else + require(modulename(program)).run(args) + end end end From dff86859de98d111799aa35e59fc3d4cc72c122b Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 4 Oct 2016 10:20:59 +0000 Subject: [PATCH 035/631] core.worker: Add execve() Worker processes now use execve() to create a totally new process image instead of using the one inherited by fork(). This is intended to simplify the relationship between parent and worker i.e. every process is an independently initialized Snabb instance. Uses the SNABB_PROGRAM_LUACODE hook to tell the worker process how to start. --- src/core/worker.lua | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index 2a5a0d1543..0206a312a2 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -1,12 +1,18 @@ -- worker.lua - Execute "worker" child processes to execute app networks -- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + -- API: -- start(name, core) -- stop(name) -- status() -> table of { name = } -- configure(name, config) +local lib = require("core.lib") +local shm = require("core.shm") +local S = require("syscall") + -------------------------------------------------------------- -- Master (parent) process code -------------------------------------------------------------- @@ -21,9 +27,14 @@ end -- The child will execute an app network when provided with configure(). function start (name, core) local pid = S.fork() - if pid ~= 0 then - -- Child (worker) process - init(name, core, S.getppid()) + if pid == 0 then + -- Lock affinity for this child + S.sched_setaffinity(0, {core}) + local env = { "SNABB_PROGRAM_LUACODE=require('core.worker').init()", + "SNABB_WORKER_NAME="..name, + "SNABB_WORKER_PARENT="..S.getppid() } + -- /proc/$$/exe is a link to the same Snabb executable that we are running + S.execve(("/proc/%d/exe"):format(S.getpid()), {}, env) else -- Parent process children[name] = { pid = pid, core = core } @@ -32,16 +43,18 @@ end -- Terminate a child process function stop (name) - S.kill(child(name), 'kill') + S.kill(child(name).pid, 'kill') end -- Return information about all worker processes in a table. function status () local status = {} for name, info in pairs(children) do + local infop = S.waitid("pid", info.pid, "nohang, exited") status[name] = { pid = info.pid, - alive = (S.kill(info.pid, 0) == 0) + core = info.core, + alive = infop.code == 0 } end return status @@ -65,14 +78,10 @@ end -- Initialize the worker by attaching to relevant shared memory -- objects and entering the main engine loop. -function init (name, core, parentpid) - local name = parameters[1] - local parent = num(parameters[2]) - local core = num(parameters[3]) - print(("Starting worker %s on core %d for parent %d"):format(name, core, parent)) - - -- Setup affinity - if core then S.sched_setaffinity(0, {core}) end +function init (name, parentpid) + local name = assert(lib.getenv("SNABB_WORKER_NAME")) + local parent = assert(lib.getenv("SNABB_WORKER_PARENT")) + print(("Starting worker %s for parent %d"):format(name, parent)) -- Create "group" alias to the shared group folder in the parent process shm.alias("group", "/"..parent.."/group") @@ -83,7 +92,7 @@ function init (name, core, parentpid) if not warned then print("waiting for configuration...") warned = true - C.usleep(1000) + S.nanosleep(0.001) end end From 1e1f889bc0f1ecc16f5383074a8b13573166b495 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 4 Oct 2016 10:24:50 +0000 Subject: [PATCH 036/631] core.worker: Added simple selftest() function This is currently very basic because not all of the worker machinery is working yet. --- src/core/worker.lua | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/core/worker.lua b/src/core/worker.lua index 0206a312a2..7878348c2e 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -116,4 +116,30 @@ function init (name, parentpid) end end +function selftest () + print("selftest: worker") + -- XXX This selftest function is very basic. Should be expanded to + -- run app networks in child processes and ensure that they work. + local workers = { "w1", "w2", "w3" } + print("Starting children") + for _, w in ipairs(workers) do + start(w, 0) + end + print("Worker status:") + for w, s in pairs(status()) do + print((" worker %s: pid=%s core=%s alive=%s"):format( + w, s.pid, s.core, s.alive)) + end + print("Stopping children") + for _, w in ipairs(workers) do + stop(w) + end + S.nanosleep(1) + print("Worker status:") + for w, s in pairs(status()) do + print((" worker %s: pid=%s core=%s alive=%s"):format( + w, s.pid, s.core, s.alive)) + end + print("selftest: done") +end From 5c1ab27f6b712827262bb6e85d1422fb02dc7170 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 5 Oct 2016 12:23:17 +0200 Subject: [PATCH 037/631] Add iterating over YANG data file Adds methods to iterate over a YANG data file and add the data to the YANG data tree. This will perform validation on the data as it's added to the data node --- src/apps/lwaftr/snabb-softwire.yang | 116 ++++++++------ src/lib/yang/helpers.lua | 228 +++++++++++++++++++++++----- src/lib/yang/schema.lua | 29 ++-- src/lib/yang/yang.lua | 150 ++++++++++++++++-- 4 files changed, 420 insertions(+), 103 deletions(-) diff --git a/src/apps/lwaftr/snabb-softwire.yang b/src/apps/lwaftr/snabb-softwire.yang index 82abd6c912..6c98f52939 100644 --- a/src/apps/lwaftr/snabb-softwire.yang +++ b/src/apps/lwaftr/snabb-softwire.yang @@ -6,7 +6,7 @@ module snabb-softwire { organization "Igalia"; contact "Jessica Tallon "; - description ""; + description "Configuration for the Snabb Switch lwAFTR."; revision 2016-08-01 { description "Initial revision."; @@ -41,9 +41,9 @@ module snabb-softwire { } description "This is a VLAN tag according to 802.1Q Ethernet tagging."; } - + container softwire-config { - description "Configuration for Snabb lwaftr."; + description "Configuration for Snabb lwaftr."; leaf ipv4-ip { type inet:ipv4-address; @@ -95,7 +95,7 @@ module snabb-softwire { description "Set the MTU value for the IPv6 NIC."; } } - + container L2-next-hop { description "One might expect to set the default gateway IPv4 and IPv6 addresses and have those resolved via ARP, however ARP is @@ -104,7 +104,7 @@ module snabb-softwire { instead specify the IP addresses of the next hops and have the lwAFTR resolve the corresponding MAC addresses via NDP and ARP."; - + leaf b4-mac { type inet:mac-address; description "Sets the MAC address of the next hop for the internal @@ -127,10 +127,10 @@ module snabb-softwire { type inet:ipv6-address; description "Sets the IPv4 address of the next hop for the internal facing NIC that will be used to resolve the MAC address."; - } + } } - - + + container rate-limiting-icmp { description "ICMP rate limiting is mandated by many RFCs. This configures the number of ICMP packets the lwAFTR can send for the given @@ -144,7 +144,7 @@ module snabb-softwire { period."; } - leaf icmpv6-rate-limiter-n-seconds { + leaf seconds { type PositiveNumber; default 2; description "The time period given in seconds."; @@ -195,18 +195,18 @@ module snabb-softwire { description "Sets the tag for v4"; } - leaf v6-vlan-tag { + leaf v6-tag { type VlanTag; description "Sets the tag for v6"; } } - + container filters { description "This configures the ingress and egress filters for each NIC. If set, these should be a pflang filter, pflang is the language of tcpdump, libpcap and other tools. The filters will run on the packets without vlan tags, if any."; - + leaf ipv4-ingress-filter { type string; description "Sets the pflang filter for IPv4 ingress."; @@ -235,70 +235,98 @@ module snabb-softwire { B4. It contains information so the lwAFTR knows which IPv4 (or part of an IPv4 address) is associated with which B4."; - list psid_map { - key "ipv4-address"; + list binding-entry { + key "binding-ipv6info"; description "Defines the set of IPv4 addresses that are provisioned by a lwAFTR. It also defines the way in which those addresses are shared by specifying the psid_length and shift. See RFC 7597 for more details on the PSID scheme for sharing IPv4 addresses."; - leaf ipv4-address { - type inet:ipv4-address; - mandatory true; - description "The IPv4 address that is provisioned"; + leaf binding-ipv6info { + type union { + type inet:ipv6-address; + type inet:ipv6-prefix; + } + mandatory false; } - leaf psid_length { - type uint16; - default 0; + leaf binding-ipv4-addr { + type inet:ipv4-address; + mandatory true; } - leaf shift { - type uint16; - default 16; + container port-set { + leaf psid-offset { + type uint8 { + range 0..16; + } + mandatory true; + } + + leaf psid-len { + type uint8 { + range 0..15; + } + mandatory true; + } + + leaf psid { + type uint16; + mandatory true; + } + } + + leaf br-ipv6-addr { + type inet:ipv6-address; + mandatory true; } - } - list br-addresses { + leaf lifetime { + type uint32; + units seconds; + } + } + + list br-addresses { key "ipv6-address"; leaf ipv6-address { type inet:ipv6-address; } - } + } - list binding-table-entry { + list binding-table-entry { key "ipv4-address psid"; leaf ipv4-address { - type inet:ipv4-address; - mandatory true; - description "Public IPv4 address of the softwire."; + type inet:ipv4-address; + mandatory true; + description "Public IPv4 address of the softwire."; } leaf psid { - type uint16; - mandatory true; - description "Port set ID"; + type uint16; + mandatory true; + description "Port set ID"; } leaf padding { - type uint16; - description "Zeros"; + type uint16; + description "Zeros"; } leaf br { - type uint32; - mandatory true; - description "Border router"; + type uint32; + mandatory true; + description "Border router"; } leaf ipv6-address { - type inet:ipv6-address; - mandatory true; - description "Address of the B4"; + type inet:ipv6-address; + mandatory true; + description "Address of the B4"; } } - } -} + } +} diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index 7b4d350c5e..fafe60d5a0 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -6,12 +6,13 @@ local ffi = require("ffi") local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") local corelib = require("core.lib") +local ethernet = require("lib.protocol.ethernet") -- Create small getter and setter wrapper for ffi structs. local FFIType = {} function FFIType:set(value) - self.value = value + self.value = convert(value) end function FFIType:get() @@ -19,6 +20,19 @@ function FFIType:get() end +-- Converts a type to a native lua type, this will only work for things we've coded +-- to convert (i.e. it's not general purpose). +function convert(value) + -- First try boolean. + if value == "true" then + return true + elseif value == "false" then + return false + else + return tonumber(value) -- returns nil if no number is found + end +end + -- This wraps the FFIType to provide it with the box name, -- Use ffi types because they will validate that numeric values are being @@ -35,24 +49,42 @@ local box_types = { uint32 = ffi.typeof("struct { uint32_t value; }"), uint64 = ffi.typeof("struct { uint64_t value; }"), decimal64 = ffi.typeof("struct { double value; }"), - boolean = ffi.typeof("struct { bool value; }"), + boolean = ffi.typeof("struct { bool value; }") } -- Iterate through the boxes and set the FFIType metatype -for _, box in pairs(box_types) do - ffi.metatype(box, {__index=FFIType}) +for n, box in pairs(box_types) do + ffi.metatype(box, {__index=FFIType,}) end -- Add inet types, found: https://tools.ietf.org/html/rfc6021 -- Should be done via yang module import but using ad-hoc method for now. -box_types["yang:zero-based-counter64"] = function () +box_types["yang:zero-based-counter64"] = function() return box_types.uint64(0) end -function create_box(leaf_type, default) +-- Add the types defined in the snabb-softwire yang module. This should be +-- handled by consumption of the 'typedef' statements in the future, however +-- for now, hard code them in. +box_types["PacketPolicy"] = function() + local enum_opts = {"allow", "deny"} + return box_types.enumeration(enum_opts) +end + +-- WARNING: This doesn't check the range as defined in the config +box_types["PositiveNumber"] = function(...) + return box_types.uint32() +end + +-- WARNING: This doesn't check the range as defined in the config +box_types["VlanTag"] = function(...) + return box_types.uint16() +end + +function create_box(leaf_type, arguments) local box = assert(box_types[leaf_type], "Unsupported type: "..leaf_type) - if box and default ~= nil then - box = box(default) + if box and arguments ~= nil then + box = box(arguments) elseif box then box = box() end @@ -72,7 +104,7 @@ function extract_nodes(schema) if v.statements then v.statements = extract_nodes(v.statements) end if nodes[v.keyword] then table.insert(nodes[v.keyword], v) - else + elseif v.keyword then nodes[v.keyword] = {v} end end @@ -80,8 +112,12 @@ function extract_nodes(schema) end local StringBox = {} -function StringBox.new() - return setmetatable({}, {__index=StringBox}) +function StringBox.new(options, default) + local strbox = setmetatable({}, {__index=StringBox}) + if default ~= nil then + strbox:set(default) + end + return strbox end function StringBox:get() @@ -91,13 +127,33 @@ end function StringBox:set(value) self.value = value end +function StringBox.get_type() return "box-string" end box_types["string"] = StringBox.new +local MacAddress = {} +function MacAddress.new(options, default) + local mac = setmetatable({}, {__index=MacAddress}) + if default ~= nil then + mac:set(default) + end + return mac +end +function MacAddress:get() + return self.box +end +function MacAddress:set(address) + self.box = assert(ethernet:pton(address)) +end +function MacAddress.get_type() return "box-inet:mac-address" end +box_types["inet:mac-address"] = MacAddress.new + local IPv4Box = {} -function IPv4Box.new(address) - local ret = {root={}} - if address then ret.box = ipv4:pton(address) end - return setmetatable(ret, {__index=IPv4Box}) +function IPv4Box.new(options, default) + local ipv4 = setmetatable({}, {__index=IPv4Box}) + if default ~= nil then + ipv4:set(default) + end + return ipv4 end function IPv4Box:get() return self.box @@ -105,13 +161,16 @@ end function IPv4Box:set(address) self.box = assert(ipv4:pton(address)) end +function IPv4Box.get_type() return "box-inet:ipv4-address" end box_types["inet:ipv4-address"] = IPv4Box.new local IPv6Box = {} -function IPv6Box.new(address) - local ret = {} - if address then ret.box = ipv6:pton(address) end - return setmetatable(ret, {__index=IPv6Box}) +function IPv6Box.new(options, default) + local ipv6 = setmetatable({}, {__index=IPv6Box}) + if default ~= nil then + ipv6:set(default) + end + return ipv6 end function IPv6Box:get() return self.box @@ -119,12 +178,17 @@ end function IPv6Box:set(address) self.box = assert(ipv6:pton(address)) end +function IPv6Box.get_type() return "box-inet:ipv6-address" end box_types["inet:ipv6-address"] = IPv6Box.new local IPv4PrefixBox = {} -function IPv4PrefixBox.new() +function IPv4PrefixBox.new(options, default) local ret = {root={}} - return setmetatable(ret, {__index=IPv4PrefixBox}) + local ipv4_prefix = setmetatable(ret, {__index=IPv4PrefixBox}) + if default ~= nil then + ipv4_prefix:set(default) + end + return ipv4_prefix end function IPv4PrefixBox:get() @@ -143,11 +207,16 @@ function IPv4PrefixBox:set(value) end self.value = {addr, prefix} end +function IPv4PrefixBox.get_type() return "box-inet:ipv4-prefix" end local IPv6PrefixBox = {} -function IPv6PrefixBox.new() +function IPv6PrefixBox.new(options, default) local ret = {root={}} - return setmetatable(ret, {__index=IPv6PrefixBox}) + local ipv6_prefix = setmetatable(ret, {__index=IPv6PrefixBox}) + if default ~= nil then + ipv6_prefix:set(default) + end + return ipv6_prefix end function IPv6PrefixBox:get() return self.value @@ -163,15 +232,21 @@ function IPv6PrefixBox:set(value) end self.value = {addr, prefix} end +function IPv6PrefixBox.get_type() return "box-inet:ipv6-prefix" end box_types["inet:ipv6-prefix"] = IPv6PrefixBox.new local Enum = {} -function Enum.new(options) +function Enum.new(options, default) local opts = {} for _, option in pairs(options) do opts[option] = option end - return setmetatable({options=opts}, {__index=Enum}) + local mt = {__index=Enum, options=options, default=default} + local initialized_enum = setmetatable({options=opts}, mt) + if default ~= nil then + initialized_enum:set(default) + end + return initialized_enum end function Enum:get() @@ -185,19 +260,20 @@ function Enum:set(value) error("Value "..value.." is not a valid option for this Enum.") end end +function Enum.get_type() return "box-enumeration" end box_types["enumeration"] = Enum.new local Union = {} -function Union.new(types) +function Union.new(types, default) local ret = {types={}} for _, name in pairs(types) do -- 9.12 specifies unions cannot contain "leafref" or "empty" if name == "empty" or name == "leafref" then error("Union type cannot contain 'empty' or 'leafref'") end - ret.types[name] = create_box(name) + ret.types[name] = create_box(name, nil, default) end - return setmetatable(ret, {__index=Union}) + return setmetatable(ret, {__index=Union, options=types, default=default}) end function Union:get() @@ -217,6 +293,7 @@ function Union:set(v) end error(("Unable to find matching type for '%s' (%s)"):format(v, type(v))) end +function Union.get_type() return "box-union" end box_types["union"] = Union.new @@ -246,7 +323,7 @@ function Container.new(base, path) if not prop then -- This could be because it's a method defined on Container. - return Container[k] + return rawget(Container, k) elseif prop.get == nil then return prop else @@ -256,6 +333,8 @@ function Container.new(base, path) }) end +function Container.get_type() return "container" end + function Container:set_template(template) rawset(self, "template", template) end @@ -263,11 +342,33 @@ function Container:get_template() return rawget(self, "template") end +local function pp(x) + if type(x) == "table" then + io.write("{") + local first = true + for k,v in pairs(x) do + if not first then + io.write(", ") + end + io.write(k.."=") + pp(v) + first = false + end + io.write("}") + elseif type(x) == "string" then + io.write(x) + else + error("Unsupported type") + end +end + + + function Container:add_item(item) local root = rawget(self, "root") local base = rawget(self, "base") local path = rawget(self, "path") - + -- Verify that the item is being added to a list type. local schema = base:get_schema(path) if schema:get_type() ~= "list" then @@ -288,7 +389,9 @@ function Container:add_item(item) -- Create a new table of item + leavesf local data = corelib.deepcopy(item) - for name, leaf in pairs(leaves) do data[name] = leaf end + for name, leaf in pairs(leaves) do + data[name] = leaf + end -- Add the data to the container. for name, _ in pairs(data) do @@ -301,12 +404,22 @@ function Container:add_item(item) end end + -- Create the key + local key = item[schema.key] + + if key == nil then + error("List item's key ("..schema.key..") cannot be null") + end + -- Add the container entry to container. - table.insert(root, con) + root[key] = con end -function Container:add_container(name) - self:add_to_root(name, Container.new(self, name)) +function Container:add_container(name, container) + if container == nil then + container = Container.new(self, name) + end + self:add_to_root(name, container) end function Container:add_to_root(key, value) @@ -315,13 +428,38 @@ function Container:add_to_root(key, value) end function Container:duplicate() - -- Produces and returns a duplicate of the container - local root = corelib.deepcopy(rawget(self, "root")) - local copy = Container.new(rawget(self, "base"), rawget(self, "path")) - rawset(copy, "root", root) - return copy + local root = rawget(self, "root") + local dup = {} + + for k, v in pairs(root) do + -- If "v" is a table with the method .get_type it retrives the type + local table_type + if type(v) == "table" and v.get_type ~= nil then table_type = v.get_type() end + if table_type == "container" then + dup[k] = v:duplicate() + elseif table_type and table_type:sub(1, 4) == "box-" then + local box_type = table_type:sub(5, table_type:len()) + local options = getmetatable(v).options + local defaults = getmetatable(v).defaults + local dup_box = create_box(box_type, options, defaults) + dup[k] = dup_box + elseif type(v) == "cdata" then + local dup_ctype = ffi.new(ffi.typeof(v)) + ffi.copy(dup_ctype, v, ffi.sizeof(v)) + dup[k] = dup_ctype + end + end + + local duplicate_container = Container.new( + rawget(self, "base"), + rawget(self, "path") + ) + rawset(duplicate_container, "root", dup) + return duplicate_container end +function Container:get_type() return "container" end + -- Functions to help testing. function asserterror(func, ...) local success, val = pcall(func, ...) @@ -365,7 +503,7 @@ function selftest() con.testbox = "8.8.8.8" assert(ipv4:ntop(con.testbox) == "8.8.8.8") asserterror(setvalue, con, "testbox", "should fail") - + -- Test the IPv4 box (both valid and invalid data). root.testbox = box_types["inet:ipv4-address"]() con.testbox = "8.8.8.8" @@ -407,4 +545,14 @@ function selftest() asserterror(setvalue, con, "testbox", "2001:db8::/129") asserterror(setvalue, con, "testbox", "2001:db8::/-1") asserterror(setvalue, con, "testbox", "FFFFF:db8::/32") + + -- Remove following when typedef is supported. + -- Test the PacketPolicy enum + root.testbox = create_box("PacketPolicy") + con.testbox = "allow" + assert(con.testbox == "allow") + con.testbox = "deny" + assert(con.testbox == "deny") + asserterror(setvalue, con, "testbox", "not allow or deny") + asserterror(setvalue, con, "testbox", "alow") end diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 81afd79f9f..0b5bc13cae 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -13,7 +13,15 @@ module(..., package.seeall) local helpers = require("lib.yang.helpers") local h = require("syscall.helpers") -Leaf = {} +local Leaf = {} +local List = {} +local Feature = {} +local Grouping = {} +local Container = {} +local Revision = {} +local List = {} +Module = {} + function Leaf.new(base, path, src) local self = setmetatable({}, {__index=Leaf, path=path}) @@ -70,6 +78,7 @@ function Leaf.new(base, path, src) if self.range then if not self.validation then self.validation = {} end self.validation[#self.validation + 1] = function(v) + v = tonumber(v) if v < self.range[1] or v > self.range[2] then self:error("Value '%s' is out of range", path, v) end @@ -101,7 +110,6 @@ function Leaf:validate_schema(schema) end -- Yang feature -local Feature = {} function Feature.new(base, path, src) local self = setmetatable({}, {__index=Feature, path=path}) @@ -131,9 +139,8 @@ function Feature:validate_schema(src) end -- Yang list -local List = {} function List.new(base, path, src) - local ret = {leaves={}} + local ret = {leaves={}, containers={}} local self = setmetatable(ret, {__index=List, path=path}) self:validate_schema(src) @@ -143,10 +150,16 @@ function List.new(base, path, src) if src.uses then self.uses = src.uses[1].argument end if src.leaf then for _, leaf in pairs(src.leaf) do - local path = self.path.."."..leaf.argument + local path = path.."."..leaf.argument self.leaves[leaf.argument] = Leaf.new(base, path, leaf.statements) end end + if src.container then + for _, container in pairs(src.container) do + local path = path.."."..container.argument + self.containers[container.argument] = Container.new(base, path, container.statements) + end + end return self end @@ -162,7 +175,6 @@ function List:validate_schema(src) end -- Yang group -local Grouping = {} function Grouping.new(base, path, src) local ret = {leaves={}} local self = setmetatable(ret, {__index = Grouping, path=path}) @@ -197,7 +209,6 @@ function Grouping:validate_schema(src) helpers.cardinality("grouping", getmetatable(self).path, cardinality, src) end -local Container = {} function Container.new(base, path, src) local ret = {leaves={}, containers={}, lists={}} local self = setmetatable(ret, {__index=Container, path=path}) @@ -256,7 +267,6 @@ function Container:validate_schema(src) end -- Yang Revision -local Revision = {} function Revision.new(base, path, src) local self = setmetatable({}, {__index=Revision, path=path}) @@ -280,7 +290,6 @@ function Revision:validate_schema(src) end -- Yang Module -Module = {} function Module.new(base, name, src) local ret = {body={}, name=name, modules={}, revisions={}, features={}, groupings={}, containers={}} @@ -483,4 +492,4 @@ function selftest() local desc = mod.containers["fruit-bowl"].leaves.description assert(desc.type == "string") assert(desc.description == "About the bowl") -end \ No newline at end of file +end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index aa64b43391..c610fde3dd 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -28,7 +28,7 @@ -- }]] -- base["example"].organization == "YANG IPv4 over IPv6" -- base["example"].containers["conf"].leaves["description"].type == "string" --- +-- -- -- Setting data -- base.data["example"]["conf"].description = "hello!" -- base.data["example"]["conf"].testvalue = "error" -- This errors. @@ -105,19 +105,19 @@ function Base:produce_data_tree(schema_node, data_node) if not schema_node then error("Module cannot be resolved") end data_node = self.data[schema_node.name] end - path = getmetatable(schema_node).path + local path = getmetatable(schema_node).path if schema_node.containers then for name, container in pairs(schema_node.containers) do local new_path = path.."."..name local new_node = Container.new(self, new_path) + local current_schema = assert(self:get_schema(new_path), "No schema at path:"..new_path) + data_node:add_to_root(name, new_node) - schema_node = self:get_schema(new_path) - data_node = new_node - self:produce_data_tree(schema_node, new_node) + self:produce_data_tree(current_schema, new_node) - -- If the container has a "uses" statement we must copy across the + -- If the container has a "uses" statement we must copy across the -- leaves from the container it references to this container. if container.uses then self:handle_use(container, data_node, path, name) @@ -127,7 +127,15 @@ function Base:produce_data_tree(schema_node, data_node) if schema_node.leaves then for name, leaf in pairs(schema_node.leaves) do - data_node:add_to_root(name, helpers.create_box(leaf.type, leaf.defalt)) + -- Certain types need extra options passing in, depending on the type those + -- need to be passed in when creating the type. + local options = nil + if leaf.type == "enumeration" then + options = leaf.enums + elseif leaf.type == "union" then + options = leaf.types + end + data_node:add_to_root(name, helpers.create_box(leaf.type, options, leaf.defalt)) end end @@ -135,17 +143,27 @@ function Base:produce_data_tree(schema_node, data_node) for name, list in pairs(schema_node.lists) do local list_path = path.."."..name local container = Container.new(self, list_path) + local current_schema = assert(self:get_schema(list_path), "No schema at path:"..list_path) data_node:add_to_root(name, container) if list.uses then local template = Container.new(self, list_path) self:handle_use(list, template, list_path, name) container:set_template(template) + else + -- Make a data container for the template + local template_container = Container.new(self, list_path) + local data_template = self:produce_data_tree( + current_schema, + template_container + ) + + container:set_template(template_container) end end end - return self.data + return data_node end function Base:schema_for_uses(schema) @@ -175,6 +193,120 @@ function Base:add(key, node) self.data[key] = node end +function Base:load_data(data, filename) + local parsed_data = parser.parse_string(data, filename) + local data = self:produce_data_tree() + + -- Function which can take a node, set any leaf values and recursively call + -- over collections, etc. + function recursively_add(node, path, parent, data_node) + for _, n in pairs(node) do + -- Create a path for the keyword argument pair. + local new_path = "" + if path == nil then + new_path = n.keyword + else + new_path = path.."."..n.keyword + end + + local schema = self:get_schema(new_path) + + if n.statements then + if schema.get_type() == "list" then + -- Lists are a special case, first we need to conver them from their + -- current format to a more useful {leafname: value, leafname: value} + -- type table. Then we want to add the item to the list parent. + local converted = {} + for _, leaf in pairs(n.statements) do + converted[leaf.keyword] = leaf.argument + end + + -- Extract key and add it to the converted. + converted[schema.key] = n.argument + + local data_node = self:find_data_node(new_path, nil, nil, true) + data_node:add_item(converted) + + recursively_add(n.statements, new_path, data_node) + else + if parent ~= nil then + local new_dn = self:find_data_node(new_path) + parent:add_container(n.keyword, new_dn) + recursively_add(n.statements, new_path, parent, new_dn) + else + recursively_add(n.statements, new_path) + end + end + else + if data_node == nil then + data_node = self:find_data_node(path) + end + data_node[n.keyword] = n.argument + end + end + end + + -- Recursively add the data + recursively_add(parsed_data) + return self.data +end + +function Base:find_data_node(schema, data_node, current_schema, raw) + -- If no data node has been provided assume we're starting from Base.data + if data_node == nil then + data_node = self.data + end + + -- If there is no current_schema we must be in our first iteration (or + -- someone has called this function incorrectly). + if current_schema == nil then + current_schema = schema + end + + -- First find the first node in the schema + local head = current_schema:match("[%w-_]*") + local tail = current_schema:sub(head:len() + 2) + + -- If it's a list we want to access the list's version as that's where all + -- the values are, the non-templated version is just the raw data. + local node + if data_node:get_template() ~= nil then + node = data_node:get_template()[head] + else + node = data_node[head] + end + + -- If the node doesn't exist, we should display a useful error. + if node == nil then + local current = nil + if current_schema then + current = current_schema + else + current = schema + end + self:error(current, head, "Can't find data node") + end + + -- Otherwise we need to check if we're at the end of the schema, if we are + -- we should return what we've found, otherwise continue to recursively call. + if tail == "" then + if raw ~= true and node:get_template() ~= nil then + return node:get_template() + else + return node + end + else + return self:find_data_node(schema, node, tail, raw) + end +end + +function Base:load_data_file(filename) + local file_in = assert(io.open(filename)) + local contents = file_in:read("*a") + file_in:close() + return self:load_data(contents, filename) +end + function load_schema(schema, filename) local parsed_yang = parser.parse_string(schema, "selftest") local base = Base.new() @@ -282,4 +414,4 @@ function selftest() -- Finally check tht validation occurs when you're adding entries with -- invalid data in them, in this case sore needs to be an integer. asserterror(bowl.contents, bowl.contents, {name="Pear", score="Good"}) -end \ No newline at end of file +end From b5163faca4a975369973a9cb9ad65770fbce8134 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 13 Oct 2016 08:51:55 +0000 Subject: [PATCH 038/631] Fix bug in reading yang data file This bug caused there to be an invalid value set on reading lists this is due to it not setting child containers of lists to the correct list item. --- src/lib/yang/helpers.lua | 6 ++++-- src/lib/yang/yang.lua | 16 +++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua index fafe60d5a0..adccab4ef1 100644 --- a/src/lib/yang/helpers.lua +++ b/src/lib/yang/helpers.lua @@ -298,8 +298,8 @@ function Union.get_type() return "box-union" end box_types["union"] = Union.new Container = {} -function Container.new(base, path) - local ret = {root={}, base=base, path=path} +function Container.new(base, path, parent) + local ret = {root={}, base=base, path=path, parent=parent} return setmetatable(ret, { __newindex = function (_, k, v) -- Validate the value prior to setting it. @@ -335,6 +335,8 @@ end function Container.get_type() return "container" end +function Container:get_parent() return rawget(self, "parent") end + function Container:set_template(template) rawset(self, "template", template) end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index c610fde3dd..3425611996 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -110,7 +110,7 @@ function Base:produce_data_tree(schema_node, data_node) if schema_node.containers then for name, container in pairs(schema_node.containers) do local new_path = path.."."..name - local new_node = Container.new(self, new_path) + local new_node = Container.new(self, new_path, data_node) local current_schema = assert(self:get_schema(new_path), "No schema at path:"..new_path) @@ -142,12 +142,12 @@ function Base:produce_data_tree(schema_node, data_node) if schema_node.lists then for name, list in pairs(schema_node.lists) do local list_path = path.."."..name - local container = Container.new(self, list_path) + local container = Container.new(self, list_path, data_node) local current_schema = assert(self:get_schema(list_path), "No schema at path:"..list_path) data_node:add_to_root(name, container) if list.uses then - local template = Container.new(self, list_path) + local template = Container.new(self, list_path, data_node) self:handle_use(list, template, list_path, name) container:set_template(template) else @@ -210,7 +210,6 @@ function Base:load_data(data, filename) end local schema = self:get_schema(new_path) - if n.statements then if schema.get_type() == "list" then -- Lists are a special case, first we need to conver them from their @@ -227,12 +226,11 @@ function Base:load_data(data, filename) local data_node = self:find_data_node(new_path, nil, nil, true) data_node:add_item(converted) - recursively_add(n.statements, new_path, data_node) + recursively_add(n.statements, new_path, data_node[n.argument]) else if parent ~= nil then - local new_dn = self:find_data_node(new_path) - parent:add_container(n.keyword, new_dn) - recursively_add(n.statements, new_path, parent, new_dn) + local dn = parent[n.keyword] + recursively_add(n.statements, new_path, parent, dn) else recursively_add(n.statements, new_path) end @@ -269,7 +267,7 @@ function Base:find_data_node(schema, data_node, current_schema, raw) -- If it's a list we want to access the list's version as that's where all -- the values are, the non-templated version is just the raw data. - local node + local node if data_node:get_template() ~= nil then node = data_node:get_template()[head] else From 0888062ee7b562f8b38874bed0dbb550b109f203 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 20 Oct 2016 17:23:07 +0000 Subject: [PATCH 039/631] Remove duplicated code in SnabbVMX selftests --- .../snabbvmx/tests/nexthop/selftest.sh | 16 +++ src/program/snabbvmx/tests/selftest.sh | 99 ++----------------- .../snabbvmx/tests/test_env/test_env.sh | 51 +++++----- 3 files changed, 49 insertions(+), 117 deletions(-) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index 349a6cc3c8..094903fcf9 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -17,6 +17,22 @@ if [[ -z "$SNABB_PCI1" ]]; then exit $SKIPPED_CODE fi +LWAFTR_IPV4_ADDRESS=10.0.1.1 +LWAFTR_IPV6_ADDRESS=fc00::100 +MAC_ADDRESS_NET0=02:AA:AA:AA:AA:AA +MIRROR_TAP=tap0 +NEXT_HOP_MAC=02:99:99:99:99:99 +NEXT_HOP_V4=10.0.1.100 +NEXT_HOP_V6=fc00::1 +SNABBVMX_DIR=program/snabbvmx +PCAP_INPUT=$SNABBVMX_DIR/tests/pcap/input +PCAP_OUTPUT=$SNABBVMX_DIR/tests/pcap/output +SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr.cfg +SNABBVMX_ID=xe1 +SNABBVMX_LOG=snabbvmx.log +SNABB_TELNET0=5000 +VHU_SOCK0=/tmp/vh1a.sock + # Load environment settings. source program/snabbvmx/tests/test_env/test_env.sh diff --git a/src/program/snabbvmx/tests/selftest.sh b/src/program/snabbvmx/tests/selftest.sh index a654cb4206..d2052032b6 100755 --- a/src/program/snabbvmx/tests/selftest.sh +++ b/src/program/snabbvmx/tests/selftest.sh @@ -15,83 +15,26 @@ if [[ -z "$SNABB_PCI1" ]]; then exit $SKIPPED_CODE fi -LWAFTR_IPV6_ADDRESS=fc00::100 LWAFTR_IPV4_ADDRESS=10.0.1.1 - -BZ_IMAGE="$HOME/.test_env/bzImage" -HUGEPAGES_FS=/dev/hugepages -IMAGE="$HOME/.test_env/qemu.img" -MAC_ADDRESS_NET0="02:AA:AA:AA:AA:AA" -MEM=1024M +LWAFTR_IPV6_ADDRESS=fc00::100 +MAC_ADDRESS_NET0=02:AA:AA:AA:AA:AA MIRROR_TAP=tap0 +NEXT_HOP_MAC=02:99:99:99:99:99 +NEXT_HOP_V4=10.0.1.100 +NEXT_HOP_V6=fc00::1 SNABBVMX_DIR=program/snabbvmx PCAP_INPUT=$SNABBVMX_DIR/tests/pcap/input PCAP_OUTPUT=$SNABBVMX_DIR/tests/pcap/output SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr.cfg SNABBVMX_ID=xe1 +SNABBVMX_LOG=snabbvmx.log SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock -SNABBVMX_LOG=snabbvmx.log -rm -f $SNABBVMX_LOG - -# Some of these functions are from program/snabbfv/selftest.sh. -# TODO: Refactor code to a common library. - -# Usage: run_telnet [] -# Runs on VM listening on telnet . Waits seconds -# for before closing connection. The default of is 2. -function run_telnet { - (echo "$2"; sleep ${3:-2}) \ - | telnet localhost $1 2>&1 -} - -# Usage: wait_vm_up -# Blocks until ping to 0::0 suceeds. -function wait_vm_up { - local timeout_counter=0 - local timeout_max=50 - echo -n "Waiting for VM listening on telnet port $1 to get ready..." - while ( ! (run_telnet $1 "ping6 -c 1 0::0" | grep "1 received" \ - >/dev/null) ); do - # Time out eventually. - if [ $timeout_counter -gt $timeout_max ]; then - echo " [TIMEOUT]" - exit 1 - fi - timeout_counter=$(expr $timeout_counter + 1) - sleep 2 - done - echo " [OK]" -} +# Load environment settings. +source program/snabbvmx/tests/test_env/test_env.sh -function qemu_cmd { - echo "qemu-system-x86_64 \ - -kernel ${BZ_IMAGE} -append \"earlyprintk root=/dev/vda rw console=tty0\" \ - -enable-kvm -drive format=raw,if=virtio,file=${IMAGE} \ - -M pc -smp 1 -cpu host -m ${MEM} \ - -object memory-backend-file,id=mem,size=${MEM},mem-path=${HUGEPAGES_FS},share=on \ - -numa node,memdev=mem \ - -chardev socket,id=char1,path=${VHU_SOCK0},server \ - -netdev type=vhost-user,id=net0,chardev=char1 \ - -device virtio-net-pci,netdev=net0,addr=0x8,mac=${MAC_ADDRESS_NET0} \ - -serial telnet:localhost:${SNABB_TELNET0},server,nowait \ - -display none" -} - -function quit_screen { screen_id=$1 - screen -X -S "$screen_id" quit &> /dev/null -} - -function run_cmd_in_screen { screen_id=$1; cmd=$2 - screen_id="${screen_id}-$$" - quit_screen "$screen_id" - screen -dmS "$screen_id" bash -c "$cmd >> $SNABBVMX_LOG" -} - -function qemu { - run_cmd_in_screen "qemu" "`qemu_cmd`" -} +rm -f $SNABBVMX_LOG function monitor { action=$1 local cmd="sudo ./snabb lwaftr monitor $action" @@ -103,30 +46,6 @@ function tcpreplay { pcap=$1; pci=$2 run_cmd_in_screen "tcpreplay" "$cmd" } -function start_test_env { - if [[ ! -f "$IMAGE" ]]; then - echo "Couldn't find QEMU image: $IMAGE" - exit $SKIPPED_CODE - fi - - # Run qemu. - qemu - - # Wait until VMs are ready. - wait_vm_up $SNABB_TELNET0 - - # Manually set ip addresses. - run_telnet $SNABB_TELNET0 "ifconfig eth0 up" >/dev/null - run_telnet $SNABB_TELNET0 "ip -6 addr add $LWAFTR_IPV6_ADDRESS/64 dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "ip addr add $LWAFTR_IPV4_ADDRESS/24 dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "ip neigh add 10.0.1.100 lladdr 02:99:99:99:99:99 dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "ip -6 neigh add fc00::1 lladdr 02:99:99:99:99:99 dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "route add default gw 10.0.1.100 eth0" >/dev/null - run_telnet $SNABB_TELNET0 "route -6 add default gw fc00::1 eth0" >/dev/null - run_telnet $SNABB_TELNET0 "sysctl -w net.ipv4.conf.all.forwarding=1" >/dev/null - run_telnet $SNABB_TELNET0 "sysctl -w net.ipv6.conf.all.forwarding=1" >/dev/null -} - function create_mirror_tap_if_needed { ip tuntap add $MIRROR_TAP mode tap 2>/dev/null ip li set dev $MIRROR_TAP up 2>/dev/null diff --git a/src/program/snabbvmx/tests/test_env/test_env.sh b/src/program/snabbvmx/tests/test_env/test_env.sh index deadc59808..c4b5ebed6e 100755 --- a/src/program/snabbvmx/tests/test_env/test_env.sh +++ b/src/program/snabbvmx/tests/test_env/test_env.sh @@ -7,26 +7,11 @@ if [[ $EUID != 0 ]]; then exit 1 fi -export LWAFTR_IPV6_ADDRESS=fc00::100/64 -export LWAFTR_IPV4_ADDRESS=10.0.1.1/24 -export BZ_IMAGE="$HOME/.test_env/bzImage" -export HUGEPAGES_FS=/dev/hugepages -export IMAGE="$HOME/.test_env/qemu.img" -export MAC_ADDRESS_NET0="02:AA:AA:AA:AA:AA" -export MEM=1024M -export MIRROR_TAP=tap0 -export SNABBVMX_DIR=program/snabbvmx -export PCAP_INPUT=$SNABBVMX_DIR/tests/pcap/input -export PCAP_OUTPUT=$SNABBVMX_DIR/tests/pcap/output -export SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr-vlan.cfg -export SNABBVMX_ID=xe1 -export SNABB_TELNET0=5000 -export VHU_SOCK0=/tmp/vh1a.sock -export SNABBVMX_LOG=snabbvmx.log - -# Usage: run_telnet [] -# Runs on VM listening on telnet . Waits seconds -# for before closing connection. The default of is 2. +BZ_IMAGE="$HOME/.test_env/bzImage" +HUGEPAGES_FS=/dev/hugepages +IMAGE="$HOME/.test_env/qemu.img" +MEM=1024M + function run_telnet { (echo "$2"; sleep ${3:-2}) \ | telnet localhost $1 2>&1 @@ -51,7 +36,6 @@ function wait_vm_up { echo " [OK]" } -# TODO: Use standard launch_qemu command. function qemu_cmd { echo "qemu-system-x86_64 \ -kernel ${BZ_IMAGE} -append \"earlyprintk root=/dev/vda rw console=tty0\" \ @@ -66,8 +50,13 @@ function qemu_cmd { -display none" } +function quit_screen { screen_id=$1 + screen -X -S "$screen_id" quit &> /dev/null +} + function run_cmd_in_screen { screen_id=$1; cmd=$2 screen_id="${screen_id}-$$" + quit_screen "$screen_id" screen -dmS "$screen_id" bash -c "$cmd >> $SNABBVMX_LOG" } @@ -89,12 +78,20 @@ function start_test_env { # Manually set ip addresses. run_telnet $SNABB_TELNET0 "ifconfig eth0 up" >/dev/null - run_telnet $SNABB_TELNET0 "ip -6 addr add $LWAFTR_IPV6_ADDRESS dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "ip addr add $LWAFTR_IPV4_ADDRESS dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "ip neigh add 10.0.1.100 lladdr 02:99:99:99:99:99 dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "ip -6 neigh add fc00::1 lladdr 02:99:99:99:99:99 dev eth0" >/dev/null - run_telnet $SNABB_TELNET0 "route add default gw 10.0.1.100 eth0" >/dev/null - run_telnet $SNABB_TELNET0 "route -6 add default gw fc00::1 eth0" >/dev/null + + # Assign lwAFTR's IPV4 and IPV6 addresses to eth0. + run_telnet $SNABB_TELNET0 "ip -6 addr add ${LWAFTR_IPV6_ADDRESS}/64 dev eth0" >/dev/null + run_telnet $SNABB_TELNET0 "ip addr add ${LWAFTR_IPV4_ADDRESS}/24 dev eth0" >/dev/null + + # Add IPv4 and IPv6 nexthop address resolution to MAC. + run_telnet $SNABB_TELNET0 "ip neigh add ${NEXT_HOP_V4} lladdr ${NEXT_HOP_MAC} dev eth0" >/dev/null + run_telnet $SNABB_TELNET0 "ip -6 neigh add ${NEXT_HOP_V6} lladdr ${NEXT_HOP_MAC} dev eth0" >/dev/null + + # Set nexthop as default gateway, both in IPv4 and IPv6. + run_telnet $SNABB_TELNET0 "route add default gw ${NEXT_HOP_V4} eth0" >/dev/null + run_telnet $SNABB_TELNET0 "route -6 add default gw ${NEXT_HOP_V6} eth0" >/dev/null + + # Activate IPv4 and IPv6 forwarding. run_telnet $SNABB_TELNET0 "sysctl -w net.ipv4.conf.all.forwarding=1" >/dev/null run_telnet $SNABB_TELNET0 "sysctl -w net.ipv6.conf.all.forwarding=1" >/dev/null } From 3cd27427eaa3cac2e784463cc44b07450bca7c13 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Fri, 21 Oct 2016 16:48:56 +0200 Subject: [PATCH 040/631] Remove the 'lwaftr generator' command (#508) * Remove the 'lwaftr generator' command * Move the 'packetblaster lwaftr' code from src/apps/ to the command directory * Add docs for using 'packetblaster lwaftr' in place of 'lwaftr generator' --- src/program/lwaftr/README | 1 - .../lwaftr/doc/README.configuration.md | 2 +- src/program/lwaftr/doc/README.running.md | 5 + src/program/lwaftr/generator/README | 18 --- src/program/lwaftr/generator/README.inc | 1 - src/program/lwaftr/generator/generator.lua | 121 ------------------ src/program/packetblaster/lwaftr/README | 40 ++++++ .../packetblaster/lwaftr/lib.lua} | 0 src/program/packetblaster/lwaftr/lwaftr.lua | 3 +- 9 files changed, 47 insertions(+), 144 deletions(-) delete mode 100644 src/program/lwaftr/generator/README delete mode 120000 src/program/lwaftr/generator/README.inc delete mode 100644 src/program/lwaftr/generator/generator.lua rename src/{apps/test/lwaftr.lua => program/packetblaster/lwaftr/lib.lua} (100%) diff --git a/src/program/lwaftr/README b/src/program/lwaftr/README index 7851a18247..83c2935f1d 100644 --- a/src/program/lwaftr/README +++ b/src/program/lwaftr/README @@ -7,7 +7,6 @@ Usage: snabb lwaftr generate-binding-table snabb lwaftr control snabb lwaftr loadtest - snabb lwaftr generator snabb lwaftr query Use --help for per-command usage. diff --git a/src/program/lwaftr/doc/README.configuration.md b/src/program/lwaftr/doc/README.configuration.md index 3cad3ca289..feb351a0d2 100644 --- a/src/program/lwaftr/doc/README.configuration.md +++ b/src/program/lwaftr/doc/README.configuration.md @@ -1,7 +1,7 @@ # Configuration The lwAFTR is configured by a text file. Where applicable, default values can -be found in [the code](../../../../apps/lwaftr/conf.lua#L72). +be found in [the code](../../../apps/lwaftr/conf.lua#L72). Here's an example: diff --git a/src/program/lwaftr/doc/README.running.md b/src/program/lwaftr/doc/README.running.md index 2eca3dfeca..40417adbf3 100644 --- a/src/program/lwaftr/doc/README.running.md +++ b/src/program/lwaftr/doc/README.running.md @@ -61,3 +61,8 @@ The load generator will push packets on the IPv4 and IPv6 interfaces, ramping up from 0 Gbps to 10 Gbps (by default). At each step it measures the return traffic from the lwAFTR, and prints out all this information to the console. The load generator stops when the test is done. + +## The packetblaster + +Another way of generating load is via the `packetblaster lwaftr` command, +see [its documentation](../../packetblaster/lwaftr/README). diff --git a/src/program/lwaftr/generator/README b/src/program/lwaftr/generator/README deleted file mode 100644 index 00b3d3ff6c..0000000000 --- a/src/program/lwaftr/generator/README +++ /dev/null @@ -1,18 +0,0 @@ -Generates IPv4 or IPv6 traffic for the softwires in a binding-table. - -Options: - - --max-packets Maximum number of packets. - --packet-size Sets total size for each packet. - --pcap Saves packets to a pcap file. - --duration Run script during num seconds. - -Usage: - -generator --from-inet lwaftr.conf start_ipv4 psid_len [PCI] -generator --from-b4 lwaftr.conf start_ipv4 start_b4 br psid_len [PCI] - -Examples: - -generator --from-inet --pcap from-inet.pcap lwaftr.conf 193.5.1.100 6 -generator --from-b4 --pcap from-b4.pcap lwaftr.conf 193.5.1.100 fc00:1:2:3:4:5:0:7e fc00::100 6 diff --git a/src/program/lwaftr/generator/README.inc b/src/program/lwaftr/generator/README.inc deleted file mode 120000 index 100b93820a..0000000000 --- a/src/program/lwaftr/generator/README.inc +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/program/lwaftr/generator/generator.lua b/src/program/lwaftr/generator/generator.lua deleted file mode 100644 index ca4dffeca9..0000000000 --- a/src/program/lwaftr/generator/generator.lua +++ /dev/null @@ -1,121 +0,0 @@ -module(..., package.seeall) - -local Intel82599 = require("apps.intel.intel_app").Intel82599 -local PcapWriter = require("apps.pcap.pcap").PcapWriter -local config = require("core.config") -local generator = require("apps.lwaftr.generator") -local lib = require("core.lib") -local lwconf = require("apps.lwaftr.conf") - -local DEFAUL_MAX_PACKETS = 10 - -function show_usage(code) - print(require("program.lwaftr.generator.README_inc")) - main.exit(code) -end - -function parse_args(args) - local opts, handlers = {}, {} - function handlers.i() - opts.from_inet = true - end - function handlers.b() - opts.from_b4 = true - end - function handlers.m(arg) - opts.max_packets = assert(tonumber(arg), "max-packets must be a number") - end - function handlers.s(arg) - opts.packet_size = assert(tonumber(arg), "packet-size must be a number") - end - function handlers.D(arg) - opts.duration = assert(tonumber(arg), "duration must be a number") - end - function handlers.p(filename) - opts.pcap = filename - end - function handlers.h() - show_usage(0) - end - args = lib.dogetopt(args, handlers, "ibm:s:D:p:h", - { ["from-inet"]="i", ["from-b4"]="b", - ["max-packets"]="m", ["packet-size"]="s", - duration="D", pcap="p", help="h" }) - return opts, args -end - -function run(args) - local opts, args = parse_args(args) - - if opts.from_inet and opts.from_b4 - or not (opts.from_inet or opts.from_b4) then - show_usage(1) - end - - local c = config.new() - - -- Set default max_packets value when printing to pcap. - if opts.pcap and not opts.max_packets then - opts.max_packets = DEFAUL_MAX_PACKETS - end - - local lwaftr_config, start_inet, psid_len, pciaddr - if opts.from_inet then - local num_args = 4 - if opts.pcap then num_args = num_args - 1 end - if #args ~= num_args then - show_usage(1) - end - lwaftr_config, start_inet, psid_len, pciaddr = unpack(args) - local conf = lwconf.load_lwaftr_config(lwaftr_config) - config.app(c, "generator", generator.from_inet, { - dst_mac = conf.aftr_mac_inet_side, - src_mac = conf.inet_mac, - start_inet = start_inet, - psid_len = psid_len, - max_packets = opts.max_packets, - num_ips = opts.num_ips, - packet_size = opts.packet_size, - vlan_tag = conf.vlan_tagging and conf.v4_vlan_tag, - }) - end - - local start_b4, br - if opts.from_b4 then - local num_args = 6 - if opts.pcap then num_args = num_args - 1 end - if #args ~= num_args then - show_usage(1) - end - lwaftr_config, start_inet, start_b4, br, psid_len, pciaddr = unpack(args) - local conf = lwconf.load_lwaftr_config(lwaftr_config) - config.app(c, "generator", generator.from_b4, { - src_mac = conf.next_hop6_mac, - dst_mac = conf.aftr_mac_b4_side, - start_inet = start_inet, - start_b4 = start_b4, - br = br, - psid_len = psid_len, - max_packets = opts.max_packets, - num_ips = opts.num_ips, - packet_size = opts.packet_size, - vlan_tag = conf.vlan_tagging and conf.v6_vlan_tag, - }) - end - - if opts.pcap then - config.app(c, "pcap", PcapWriter, opts.pcap) - config.link(c, "generator.output -> pcap.input") - opts.duration = opts.duration or 1 - else - config.app(c, "nic", Intel82599, { pciaddr = pciaddr }) - config.link(c, "generator.output -> nic.rx") - end - - engine.configure(c) - if opts.duration then - engine.main({ duration = opts.duration }) - else - engine.main({ noreport = true }) - end -end diff --git a/src/program/packetblaster/lwaftr/README b/src/program/packetblaster/lwaftr/README index c575470d5a..4c452a1db3 100644 --- a/src/program/packetblaster/lwaftr/README +++ b/src/program/packetblaster/lwaftr/README @@ -65,6 +65,7 @@ Usage: packetblaster lwaftr [OPTIONS] -h, --help Print usage information. + This tool generates two types of traffic according to RFC7596: - B4 IPv4-in-IPv6 traffic towards lwAFTR (left to right) @@ -77,6 +78,7 @@ Topology From RFC7596, Figure 1: +--------+ +---------+ +------+ +-------------+ lw4o6 NAPT model: subscriber state in the lwAFTR, NAPT state in the lwB4 + Example 1: Measure performance of single stick LWAFTR (handling IPv4 and IPv6 traffic over a single interface). Packetblaster lwaftr generates 50% IPv4 and 50% IPv6 encapsulated traffic of IMIX line traffic: @@ -100,6 +102,7 @@ traffic over a single interface). Packetblaster lwaftr generates 50% IPv4 and v6+v4: 1.574+1.574 = 3.148 MPPS, 4.950+4.446 = 9.395 Gbps, lost 0 pkts v6+v4: 1.574+1.574 = 3.148 MPPS, 4.950+4.446 = 9.395 Gbps, lost 0 pkts + Example 2: Uni-directional througput test with two instances. Run one client1 with 0 MPPS and on client2 on the other end of the wire with the desired packet rate. The first instance will report on received @@ -145,6 +148,7 @@ traffic: v6+v4: 0.000+0.000 = 0.000 MPPS, 0.000+0.000 = 0.000 Gbps, lost 0 pkts v6+v4: 0.000+0.000 = 0.000 MPPS, 0.000+0.000 = 0.000 Gbps, lost 0 pkts + Example with Linux tap interface: $ sudo ip tuntap add dev tap0 mode tap @@ -169,3 +173,39 @@ Example with Linux tap interface: 14:04:43.573161 00:00:00:00:00:00 > 00:00:00:00:00:00, ethertype IPv4 (0x0800), length 608: 8.8.8.8.12345 > 0.0.0.0.1024: UDP, length 558 14:04:43.573297 00:00:00:00:00:00 > 00:00:00:00:00:00, ethertype IPv4 (0x0800), length 1514: 8.8.8.8.12345 > 0.0.0.0.1024: UDP, length 1464 14:04:43.573430 00:00:00:00:00:00 > 00:00:00:00:00:00, ethertype IPv4 (0x0800), length 78: 8.8.8.8.12345 > 0.0.0.0.1024: UDP, length 28 + + +Example of using it in place of the removed `lwaftr generator` command: + + $ sudo ./snabb packetblaster lwaftr \ + --src_mac 02:02:02:02:02:02 --dst_mac 02:42:df:27:05:00 \ + --b4 2001:db8::40,193.5.1.100,1024 --aftr 2001:db8:ffff::100 \ + --count 60000 --pcap lwaftr-traffic.pcap --size 550 + packetblaster lwaftr: Sending 60000 clients at 1.000 MPPS to lwaftr-traffic.pcap + + IPv6: 2001:db8::40 > 2001:db8:ffff::100: 193.5.1.100:1024 > 8.8.8.8:12345 + source IPv6 and source IPv4/Port adjusted per client + IPv6 packet sizes: 590 + + IPv4: 8.8.8.8:12345 > 193.5.1.100:1024 + destination IPv4 and Port adjusted per client + IPv4 packet sizes: 550 + v6+v4: 0.000+0.000 = 0.000000 MPPS, 0.000+0.000 = 0.000000 Gbps, lost nan% + v6+v4: 0.000+0.000 = 0.000000 MPPS, 0.000+0.000 = 0.000000 Gbps, lost nan% + v6+v4: 0.000+0.000 = 0.000000 MPPS, 0.000+0.000 = 0.000000 Gbps, lost nan% + v6+v4: 0.000+0.000 = 0.000000 MPPS, 0.000+0.000 = 0.000000 Gbps, lost nan% + v6+v4: 0.000+0.000 = 0.000000 MPPS, 0.000+0.000 = 0.000000 Gbps, lost nan% + v6+v4: 0.000+0.000 = 0.000000 MPPS, 0.000+0.000 = 0.000000 Gbps, lost nan% + v6+v4: 0.000+0.000 = 0.000000 MPPS, 0.000+0.000 = 0.000000 Gbps, lost nan% + generated 60000 packets + +To split out traffic, postprocess the generated .pcap file with tcpdump: + + $ tcpdump "ip" -r lwaftr-traffic.pcap -w from-inet.pcap + reading from file lwaftr-traffic.pcap, link-type EN10MB (Ethernet) + +To generate VLAN packets, postprocess the .pcap file with tcpreplay: + + $ tcprewrite --enet-vlan=add --enet-vlan-pri=0 \ + --enet-vlan-cfi=0 --enet-vlan-tag=444 \ + --infile=from-inet.pcap --outfile=from-inet-vlan.pcap diff --git a/src/apps/test/lwaftr.lua b/src/program/packetblaster/lwaftr/lib.lua similarity index 100% rename from src/apps/test/lwaftr.lua rename to src/program/packetblaster/lwaftr/lib.lua diff --git a/src/program/packetblaster/lwaftr/lwaftr.lua b/src/program/packetblaster/lwaftr/lwaftr.lua index bcb3dbf9cf..c22e5d831d 100644 --- a/src/program/packetblaster/lwaftr/lwaftr.lua +++ b/src/program/packetblaster/lwaftr/lwaftr.lua @@ -8,13 +8,12 @@ local timer = require("core.timer") local pci = require("lib.hardware.pci") local main = require("core.main") local S = require("syscall") -local Lwaftrgen = require("apps.test.lwaftr").Lwaftrgen +local Lwaftrgen = require("program.packetblaster.lwaftr.lib").Lwaftrgen local Tap = require("apps.tap.tap").Tap local raw = require("apps.socket.raw") local pcap = require("apps.pcap.pcap") local VhostUser = require("apps.vhost.vhost_user").VhostUser local lib = require("core.lib") -local ffi = require("ffi") local usage = require("program.packetblaster.lwaftr.README_inc") From 9c4c31b434b6398f19f9b3f81587602227a62ece Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 24 Oct 2016 09:56:29 +0200 Subject: [PATCH 041/631] Build yang files into "snabb" binary This change to the makefile will build yang files into the snabb binary, in the same way as *.inc files. The yang files are stored as text. Any .yang file in the source tree, like src/lib/yang/example.yang, will be available via e.g. "require('lib.yang.example_yang')". --- src/Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 467b88e9eb..58e8ec84ef 100644 --- a/src/Makefile +++ b/src/Makefile @@ -22,6 +22,7 @@ RMSRC = $(shell find . -name '*.md' -not -regex './obj.*' -printf '%P ') PROGRAM = $(shell find program -regex '^[^/]+/[^/]+' -type d -printf '%P ') # sort to eliminate potential duplicate of programs.inc INCSRC = $(sort $(shell find . -regex '[^\#]*\.inc' -printf '%P ') programs.inc) +YANGSRC= $(shell find . -regex '[^\#]*\.yang' -printf '%P ') LUAOBJ := $(patsubst %.lua,obj/%_lua.o,$(LUASRC)) PFLUAOBJ := $(patsubst %.lua,obj/%_lua.o,$(PFLUASRC)) @@ -33,6 +34,7 @@ JITOBJS:= $(patsubst %,obj/jit_%.o,$(JITSRC)) EXTRAOBJS := obj/jit_tprof.o obj/jit_vmprof.o obj/strict.o RMOBJS := $(patsubst %,obj/%,$(RMSRC)) INCOBJ := $(patsubst %.inc,obj/%_inc.o, $(INCSRC)) +YANGOBJ:= $(patsubst %.yang,obj/%_yang.o, $(YANGSRC)) EXE := bin/snabb $(patsubst %,bin/%,$(PROGRAM)) # TESTMODS expands to: @@ -49,7 +51,7 @@ TESTSCRIPTS = $(shell find . -name "selftest.sh" -executable | xargs) PATH := ../lib/luajit/usr/local/bin:$(PATH) -snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(INCOBJ) $(LUAJIT_A) +snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(INCOBJ) $(YANGOBJ) $(LUAJIT_A) $(E) "LINK $@" $(Q) $(CC) $(DEBUG) -Wl,--no-as-needed -Wl,-E -Werror -Wall -o $@ $^ \ ../lib/luajit/src/libluajit.a \ @@ -162,6 +164,13 @@ $(INCOBJ): obj/%_inc.o: %.inc Makefile | $(OBJDIR) echo "]=============]") > $(basename $@).luainc $(Q) luajit -bg -n $(subst /,.,$*)_inc $(basename $@).luainc $@ +$(YANGOBJ): obj/%_yang.o: %.yang Makefile | $(OBJDIR) + $(E) "YANG $@" + @(echo -n "return [=============["; \ + cat $<; \ + echo "]=============]") > $(basename $@).luayang + $(Q) luajit -bg -n $(subst /,.,$*)_yang $(basename $@).luayang $@ + # Create list of programs that exist programs.inc: program @(for d in program/*/; do basename $$d; done) > $@ From 6da1cbdce7d31d1d4ffbe49276319c749fbf19ed Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 24 Oct 2016 10:22:32 +0200 Subject: [PATCH 042/631] Fix typo in yang.lua * src/lib/yang/yang.lua (Base): Fix typo. --- src/lib/yang/yang.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 3425611996..489f39f510 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -135,7 +135,7 @@ function Base:produce_data_tree(schema_node, data_node) elseif leaf.type == "union" then options = leaf.types end - data_node:add_to_root(name, helpers.create_box(leaf.type, options, leaf.defalt)) + data_node:add_to_root(name, helpers.create_box(leaf.type, options, leaf.default)) end end From c192058f63577bcb895350ede1f80da67b9bb6ab Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 24 Oct 2016 10:40:13 +0200 Subject: [PATCH 043/631] Yang parser selftest uses bundled yang files --- src/lib/yang/parser.lua | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index b162c14ade..43ee870c11 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -355,12 +355,5 @@ function selftest() test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type"}}}}) - -- We need to locate the yang example code in a reliable way, we can't - -- give the path relative to the executable as the current working - -- directly could be different. To do this we find the absolute path to - -- the snabb executable and then build the path to the yang file from that. - local S = require("syscall") - local snabb_exe = S.readlink("/proc/self/exe"):gsub("(.-)[%w_-]*$", "%1") - local yang_example = snabb_exe.."lib/yang/example.yang" - parse_file(yang_example) + parse_file(require('lib.yang.example_yang')) end From 4be802bb216dddf9592507f5d9887d9f9aad6fd0 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 24 Oct 2016 12:29:44 +0200 Subject: [PATCH 044/631] Remove hardcoded constants in nh_fwd --- src/apps/lwaftr/nh_fwd.lua | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index d10fa63d5d..27a82fc22f 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -40,11 +40,9 @@ nh_fwd6 = { } local ethernet_header_size = constants.ethernet_header_size -local n_ether_hdr_size = 14 local n_ethertype_ipv4 = constants.n_ethertype_ipv4 -local n_ipencap = 4 -local n_ipfragment = 44 -local n_ipv4_hdr_size = 20 +local proto_ipv4 = constants.proto_ipv4 +local ipv6_frag = constants.ipv6_frag local o_ipv4_checksum = constants.o_ipv4_checksum local o_ipv4_dst_addr = constants.o_ipv4_dst_addr local o_ipv4_src_addr = constants.o_ipv4_src_addr @@ -95,6 +93,10 @@ end local function copy_ipv6(dst, src) ffi.copy(dst, src, 16) end +local function get_ipv4_header_length(ptr) + local ver_and_ihl = ptr[0] + return lshift(band(ver_and_ihl, 0xf), 2) +end -- Set a bogus source IP address fe80::, so we can recognize it later when -- it comes back from the VM. @@ -121,6 +123,7 @@ local function send_ipv4_cache_trigger(r, pkt, mac) -- Set a bogus source IP address of 0.0.0.0. local ether_dhost = get_ether_dhost_ptr(pkt) local ipv4_hdr = get_ethernet_payload(pkt) + local ipv4_hdr_size = get_ipv4_header_length(ipv4_header) local ipv4_src_ip = get_ipv4_src_ptr(ipv4_hdr) local ipv4_checksum = get_ipv4_checksum_ptr(ipv4_hdr) @@ -130,7 +133,7 @@ local function send_ipv4_cache_trigger(r, pkt, mac) -- Clear checksum to recalculate it with new source IPv4 address. wr16(ipv4_checksum, 0) - wr16(ipv4_checksum, htons(ipsum(pkt.data + n_ether_hdr_size, n_ipv4_hdr_size, 0))) + wr16(ipv4_checksum, htons(ipsum(pkt.data + ethernet_header_size, ipv4_hdr_size, 0))) transmit(r, pkt) end @@ -281,7 +284,7 @@ function nh_fwd6:push () local ipv6_header = get_ethernet_payload(pkt) local proto = get_ipv6_next_header(ipv6_header) - if proto == n_ipencap or proto == n_ipfragment then + if proto == proto_ipv4 or proto == ipv6_frag then transmit(output_service, pkt) elseif output_vm then transmit(output_vm, pkt) From 8310b87abda1fcb7fba96b663dad42b07ca7a1ed Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 24 Oct 2016 17:46:49 +0200 Subject: [PATCH 045/631] Also run selftest in dasl files --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 467b88e9eb..ccb295e3bc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -38,7 +38,7 @@ EXE := bin/snabb $(patsubst %,bin/%,$(PROGRAM)) # TESTMODS expands to: # core.memory core.lib ... # for each module that has a top-level selftest () function. -TESTMODS = $(shell find . -regex '[^\#]*\.lua' -printf '%P ' | \ +TESTMODS = $(shell find . -regex '[^\#]*\.\(lua\|dasl\)' -printf '%P ' | \ xargs grep -s -l '^function selftest *[[:punct:]]' | \ sed -e 's_\.lua__' -e 's_/_._g') From aed5d4e17c85f7c5d89e27448f6fd0f0c310c392 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 24 Oct 2016 18:35:29 +0200 Subject: [PATCH 046/631] New schema parser --- src/lib/yang/schema2.lua | 682 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 682 insertions(+) create mode 100644 src/lib/yang/schema2.lua diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua new file mode 100644 index 0000000000..5f7f7afc1a --- /dev/null +++ b/src/lib/yang/schema2.lua @@ -0,0 +1,682 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +-- This module implements the schema tree and validation for YANG. It represents +-- the YANG statements with lua tables and provides a fast but flexible way to +-- represent and validate statements. +-- +-- Since YANG statements are encapsulated in modules at the highest level one +-- should take their pre-parsed YANG document containing the module and load it +-- into the Module table. +module(..., package.seeall) +local parser = require("lib.yang.parser") + +local ffi = require("ffi") +ffi.cdef("long long atoll(const char *nptr);") +local function tointeger(str) + local i = ffi.C.atoll(str) + if tostring(i) == str.."LL" then + if i == tonumber(i) then return tonumber(i) else return i end + end +end + +local function error_with_path(path, msg, ...) + error(string.format("%s: "..msg, path, ...)) +end +local function assert_with_path(expr, path, msg, ...) + if not expr then error_with_path(path, msg, ...) end + return expr +end + +-- (kind -> (function(Node) -> value)) +local initializers = {} +local function declare_initializer(init, ...) + for _, keyword in ipairs({...}) do initializers[keyword] = init end +end + +local Node = {} +local function parse_node(src, parent_path, order) + local ret = {} + ret.kind = assert(src.keyword, 'missing keyword') + if parent_path then + ret.path = parent_path..'.'..ret.kind + else + ret.path = ret.kind + end + ret.order = order + ret.argument_string = src.argument + ret.children = parse_children(src, ret.path) + ret = setmetatable(ret, {__index=Node}) + local initialize = initializers[ret.kind] + if initialize then initialize(ret) end + return ret +end + +function parse_children(src, parent_path) + local ret = {} + for i, statement in ipairs(src.statements or {}) do + local child = parse_node(statement, parent_path, i) + if not ret[child.kind] then ret[child.kind] = {} end + table.insert(ret[child.kind], child) + end + return ret +end + +local function require_argument(node) + return assert_with_path(node.argument_string, node.path, + 'missing argument') +end + +local function parse_range(node, range) + local function parse_part(part) + local l, r = part:match("^%s*([^%.]*)%s*%.%.%s*([^%s]*)%s*$") + assert_with_path(l, node.path, 'bad range component: %s', part) + if l ~= 'min' then + l = assert_with_path(tointeger(l), node.path, "bad integer: %s", l) + end + if r ~= 'max' then + r = assert_with_path(tointeger(r), node.path, "bad integer: %s", r) + end + return { l, r } + end + local parts = range:split("|") + local res = {'or'} + for part in range:split("|") do table.insert(res, parse_part(part)) end + if #res == 1 then error_with_path(node.path, "empty range", range) + elseif #res == 2 then return res[2] + else return res end +end + +local function collect_children(node, kinds) + if type(kinds) == 'string' then return collect_children(node, {kinds}) end + local ret = {} + for _, kind in ipairs(kinds) do + if node.children[kind] then + for _, child in pairs(node.children[kind]) do + table.insert(ret, child) + end + end + end + return ret +end + +local function collect_children_by_prop(node, kinds, prop) + local ret = {} + for _, child in ipairs(collect_children(node, kinds)) do + assert_with_path(child[prop], node.path, + 'child of kind %s missing prop %s', child.kind, prop) + assert_with_path(not ret[child[prop]], node.path, + 'duplicate %s: %s', prop, child[prop]) + ret[child[prop]] = child + end + return ret +end + +local function collect_children_by_id(node, kinds) + return collect_children_by_prop(node, kinds, 'id') +end + +local function collect_body_children(node) + return collect_children_by_id( + node, + {'extension', 'feature', 'identity', 'typedef', 'grouping', + 'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', + 'anyxml', 'rpc', 'notification', 'deviation'}) +end + +local function collect_data_children(node) + return collect_children_by_id( + node, + {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml'}) +end + +local function at_least_one(tab) + for k, v in pairs(tab) do return true end + return false +end + +local function collect_data_children_at_least_1(node) + local ret = collect_data_children(node) + if not at_least_one(ret) then + error_with_path(node.path, "missing data statements") + end + return ret +end + +local function collect_data_or_case_children_at_least_1(node) + local ret = collect_children_by_id( + node, + {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', + 'anyxml', 'case'}) + if not at_least_one(ret) then + error_with_path(node.path, "missing data statements") + end + return ret +end + +local function collect_child_properties(node, kind, field) + local ret = {} + for _, child in ipairs(collect_children(node, kind)) do + table.insert(ret, child[field]) + end + return ret +end + +local function maybe_child(node, kind) + local children = collect_children(node, kind) + if #children > 1 then + error_with_path(node.path, 'expected at most one child of type %s', kind) + end + return children[1] +end + +local function maybe_child_property(node, kind, prop) + local child = maybe_child(node, kind) + if child then return child[prop] end +end + +local function require_child(node, kind) + local child = maybe_child(node, kind) + if child then return child end + error_with_path(node.path, 'missing child of type %s', kind) +end + +local function require_child_property(node, kind, prop) + return require_child(node, kind)[prop] +end + +-- Simple statement kinds with string, natural, or boolean values all +-- just initialize by parsing their argument and storing it as the +-- "value" property in the schema node. +local function init_string(node) + node.value = require_argument(node) +end +local function init_natural(node) + local arg = require_argument(node) + local as_num = tonumber(arg) + assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, + node.path, 'not a natural number: %s', arg) + node.value = as_num +end +local function init_boolean(node) + local arg = require_argument(node) + if arg == 'true' then node.value = true + elseif arg == 'false' then node.value = false + else error_with_path(node.path, 'not a valid boolean: %s', arg) end +end + +-- For all other statement kinds, we have custom initializers that +-- parse out relevant sub-components and store them as named +-- properties on the schema node. +local function init_anyxml(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_argument(node) + node.id = require_argument(node) + node.yin_element = maybe_child_property(node, 'yin-element', 'value') +end +local function init_augment(node) + node.node_id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.data = collect_data_or_case_children_at_least_1(node) +end +local function init_belongs_to(node) + node.id = require_argument(node) + node.prefix = require_child(node, 'prefix').value +end +local function init_case(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.data = collect_data_children(node) +end +local function init_choice(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.default = maybe_child_property(node, 'default', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.data = collect_children_by_id( + node, + {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) +end +local function init_container(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.presence = maybe_child_property(node, 'presence', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.data = collect_data_children(node) +end +local function init_extension(node) + node.id = require_argument(node) + node.argument = maybe_child_property(node, 'argument', 'id') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_feature(node) + node.id = require_argument(node) + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_grouping(node) + node.id = require_argument(node) + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.data = collect_data_children(node) +end +local function init_identity(node) + node.id = require_argument(node) + node.base = maybe_child_property(node, 'base', 'id') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_import(node) + node.id = require_argument(node) + node.prefix = require_child_property(node, 'prefix', 'value') +end +local function init_include(node) + node.id = require_argument(node) + node.revision_date = maybe_child_property(node, 'revision-date', 'value') +end +local function init_input(node) + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.data = collect_data_children_at_least_1(node) +end +local function init_leaf(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.type = require_child(node, 'type') + node.units = maybe_child_property(node, 'units', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.default = maybe_child_property(node, 'default', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_leaf_list(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.type = require_child(node, 'type') + node.units = maybe_child_property(node, 'units', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.min_elements = maybe_child_property(node, 'min-elements', 'value') + node.max_elements = maybe_child_property(node, 'max-elements', 'value') + node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_length(node) + -- TODO: parse length arg str + node.value = require_argument(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_list(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.key = maybe_child_property(node, 'key', 'value') + node.unique = collect_child_properties(node, 'unique', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.min_elements = maybe_child_property(node, 'min-elements', 'value') + node.max_elements = maybe_child_property(node, 'max-elements', 'value') + node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.data = collect_data_children_at_least_1(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_module(node) + node.id = require_argument(node) + node.yang_version = maybe_child_property(node, 'yang-version', 'value') + node.namespace = require_child_property(node, 'namespace', 'value') + node.prefix = require_child_property(node, 'prefix', 'value') + node.imports = collect_children_by_id(node, 'import') + node.includes = collect_children(node, 'include') + node.organization = maybe_child_property(node, 'organization', 'value') + node.contact = maybe_child_property(node, 'contact', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.revisions = collect_children(node, 'revision') + node.augments = collect_children(node, 'augment') + node.body = collect_body_children(node) +end +local function init_namespace(node) + -- TODO: parse uri? + node.value = require_argument(node) +end +local function init_notification(node) + node.id = require_argument(node) + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.data = collect_data_children(node) +end +local function init_output(node) + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.data = collect_data_children_at_least_1(node) +end +local function init_path(node) + -- TODO: parse path string + node.value = require_argument(node) +end +local function init_pattern(node) + node.value = require_argument(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_range(node) + -- TODO: parse range string + node.value = parse_range(node, require_argument(node)) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_refine(node) + node.node_id = require_argument(node) + -- All subnode kinds. + node.must = collect_child_properties(node, 'must', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + -- Containers. + node.presence = maybe_child_property(node, 'presence', 'value') + -- Leaves, choice, and (for mandatory) anyxml. + node.default = maybe_child_property(node, 'default', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + -- Leaf lists and lists. + node.min_elements = maybe_child_property(node, 'min-elements', 'value') + node.max_elements = maybe_child_property(node, 'max-elements', 'value') +end +local function init_revision(node) + -- TODO: parse date + node.value = require_argument(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_rpc(node) + node.id = require_argument(node) + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.input = maybe_child(node, 'input') + node.output = maybe_child(node, 'output') +end +local function init_type(node) + node.id = require_argument(node) + node.range = maybe_child(node, 'range') + node.fraction_digits = maybe_child_property(node, 'fraction-digits', 'value') + node.length = maybe_child_property(node, 'length', 'value') + node.patterns = collect_children(node, 'pattern') + node.enums = collect_children(node, 'enum') + -- !!! path + node.leafref = maybe_child_property(node, 'path', 'value') + node.require_instances = collect_children(node, 'require-instance') + node.identityref = maybe_child_property(node, 'base', 'value') + node.union = collect_children(node, 'type') + node.bits = collect_children(node, 'bit') +end +local function init_submodule(node) + node.id = require_argument(node) + node.yang_version = maybe_child_property(node, 'yang-version', 'value') + node.belongs_to = require_child(node, 'belongs-to') + node.imports = collect_children_by_id(node, 'import') + node.includes = collect_children_by_id(node, 'include') + node.organization = maybe_child_property(node, 'organization', 'value') + node.contact = maybe_child_property(node, 'contact', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.revisions = collect_children(node, 'revision') + node.augments = collect_children(node, 'augment') + node.body = collect_body_children(node) +end +local function init_typedef(node) + node.id = require_argument(node) + node.type = require_child(node, 'type') + node.units = maybe_child_property(node, 'units', 'value') + node.default = maybe_child_property(node, 'default', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_uses(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.refines = collect_children(node, 'refine') + node.augments = collect_children(node, 'augment') +end +local function init_value(node) + local arg = require_argument(node) + local as_num = tonumber(arg) + assert_with_path(as_num and math.floor(as_num) == as_num, + node.path, 'not an integer: %s', arg) + node.value = as_num +end + +declare_initializer( + init_string, 'prefix', 'organization', 'contact', 'description', + 'reference', 'units', 'revision-date', 'base','if-feature', + 'default', 'enum', 'bit', 'status', 'presence', 'ordered-by', 'must', + 'error-message', 'error-app-tag', 'max-value', 'key', 'unique', 'when', + 'deviation', 'deviate') +declare_initializer( + init_natural, 'yang-version', 'fraction-digits', 'position', + 'min-elements', 'max-elements') +declare_initializer( + init_boolean, 'config', 'mandatory', 'require-instance', 'yin-element') +declare_initializer(init_anyxml, 'anyxml') +declare_initializer(init_argument, 'argument') +declare_initializer(init_augment, 'augment') +declare_initializer(init_belongs_to, 'belongs-to') +declare_initializer(init_case, 'case') +declare_initializer(init_choice, 'choice') +declare_initializer(init_container, 'container') +declare_initializer(init_extension, 'extension') +declare_initializer(init_feature, 'feature') +declare_initializer(init_grouping, 'grouping') +declare_initializer(init_identity, 'identity') +declare_initializer(init_import, 'import') +declare_initializer(init_include, 'include') +declare_initializer(init_input, 'input') +declare_initializer(init_leaf, 'leaf') +declare_initializer(init_leaf_list, 'leaf-list') +declare_initializer(init_length, 'length') +declare_initializer(init_list, 'list') +declare_initializer(init_module, 'module') +declare_initializer(init_namespace, 'namespace') +declare_initializer(init_notification, 'notification') +declare_initializer(init_output, 'output') +declare_initializer(init_path, 'path') +declare_initializer(init_pattern, 'pattern') +declare_initializer(init_range, 'range') +declare_initializer(init_refine, 'refine') +declare_initializer(init_revision, 'revision') +declare_initializer(init_rpc, 'rpc') +declare_initializer(init_submodule, 'submodule') +declare_initializer(init_type, 'type') +declare_initializer(init_typedef, 'typedef') +declare_initializer(init_uses, 'uses') +declare_initializer(init_value, 'value') + +local function schema_from_ast(ast) + assert(#ast == 1, 'expected a single module form') + local parsed = parse_node(ast[1]) + assert(parsed.kind == 'module', 'not a yang module') + return parsed +end + +function parse_schema(src, filename) + return schema_from_ast(parser.parse_string(src, filename)) +end + +function parse_schema_file(filename) + return schema_from_ast(parser.parse_file(filename)) +end + +function selftest() + local test_schema = [[module fruit { + namespace "urn:testing:fruit"; + prefix "fruit"; + + import ietf-inet-types {prefix inet; } + import ietf-yang-types {prefix yang; } + + organization "Fruit Inc."; + + contact "John Smith fake@person.tld"; + + description "Module to test YANG schema lib"; + + revision 2016-05-27 { + description "Revision 1"; + reference "tbc"; + } + + revision 2016-05-28 { + description "Revision 2"; + reference "tbc"; + } + + feature bowl { + description "A fruit bowl"; + reference "fruit-bowl"; + } + + grouping fruit { + description "Represets a piece of fruit"; + + leaf name { + type string; + mandatory true; + description "Name of fruit."; + } + + leaf score { + type uint8 { + range 0..10; + } + mandatory true; + description "How nice is it out of 10"; + } + + leaf tree-grown { + type boolean; + description "Is it grown on a tree?"; + } + } + + container fruit-bowl { + description "Represents a fruit bowl"; + + leaf description { + type string; + description "About the bowl"; + } + + list contents { + uses fruit; + } + } + }]] + + -- Convert the schema using the already tested parser. + local schema = parse_schema(test_schema, "schema selftest") + + assert(schema.id == "fruit") + assert(schema.namespace == "urn:testing:fruit") + assert(schema.prefix == "fruit") + assert(schema.contact == "John Smith fake@person.tld") + assert(schema.organization == "Fruit Inc.") + assert(schema.description == "Module to test YANG schema lib") + + -- Check both modules exist. (Also need to check they've loaded) + assert(schema.imports["ietf-inet-types"]) + assert(schema.imports["ietf-yang-types"]) + + -- Check all revisions are accounted for. + assert(schema.revisions[1].description == "Revision 1") + assert(schema.revisions[1].value == "2016-05-27") + assert(schema.revisions[2].description == "Revision 2") + assert(schema.revisions[2].value == "2016-05-28") + + -- Check that the feature statements are there. + assert(schema.body["bowl"]) + assert(schema.body["bowl"].kind == 'feature') + + -- Check the groupings + assert(schema.body["fruit"]) + assert(schema.body["fruit"].description) + assert(schema.body["fruit"].data["name"]) + assert(schema.body["fruit"].data["name"].kind == "leaf") + assert(schema.body["fruit"].data["name"].type.id == "string") + assert(schema.body["fruit"].data["name"].mandatory == true) + assert(schema.body["fruit"].data["name"].description == "Name of fruit.") + assert(schema.body["fruit"].data["score"].type.id == "uint8") + assert(schema.body["fruit"].data["score"].mandatory == true) + assert(schema.body["fruit"].data["score"].type.range.value[1] == 0) + assert(schema.body["fruit"].data["score"].type.range.value[2] == 10) + + -- Check the containers description (NOT the leaf called "description") + assert(schema.body["fruit-bowl"].description == "Represents a fruit bowl") + + -- Check the container has a leaf called "description" + local desc = schema.body["fruit-bowl"].data['description'] + assert(desc.type.id == "string") + assert(desc.description == "About the bowl") + + parse_schema(require('lib.yang.example_yang')) +end From dc2d4f2e9aa4adeb645f2276f0faaf454e872313 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 25 Oct 2016 10:25:16 +0200 Subject: [PATCH 047/631] Uniformly use "body" for contained statements * src/lib/yang/schema2.lua: Change "data" members to be "body", collect submodules in "module", and collect imports by prefix. Adapt tests. --- src/lib/yang/schema2.lua | 45 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua index 5f7f7afc1a..05fb3c11e1 100644 --- a/src/lib/yang/schema2.lua +++ b/src/lib/yang/schema2.lua @@ -228,7 +228,7 @@ local function init_augment(node) node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.data = collect_data_or_case_children_at_least_1(node) + node.body = collect_data_or_case_children_at_least_1(node) end local function init_belongs_to(node) node.id = require_argument(node) @@ -241,7 +241,7 @@ local function init_case(node) node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.data = collect_data_children(node) + node.body = collect_data_children(node) end local function init_choice(node) node.id = require_argument(node) @@ -255,7 +255,7 @@ local function init_choice(node) node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.data = collect_children_by_id( + node.body = collect_children_by_id( node, {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) end @@ -271,7 +271,7 @@ local function init_container(node) node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.data = collect_data_children(node) + node.body = collect_data_children(node) end local function init_extension(node) node.id = require_argument(node) @@ -294,7 +294,7 @@ local function init_grouping(node) node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.data = collect_data_children(node) + node.body = collect_data_children(node) end local function init_identity(node) node.id = require_argument(node) @@ -314,7 +314,7 @@ end local function init_input(node) node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.data = collect_data_children_at_least_1(node) + node.body = collect_data_children_at_least_1(node) end local function init_leaf(node) node.id = require_argument(node) @@ -365,7 +365,7 @@ local function init_list(node) node.status = maybe_child_property(node, 'status', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.data = collect_data_children_at_least_1(node) + node.body = collect_data_children_at_least_1(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end @@ -374,7 +374,8 @@ local function init_module(node) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.namespace = require_child_property(node, 'namespace', 'value') node.prefix = require_child_property(node, 'prefix', 'value') - node.imports = collect_children_by_id(node, 'import') + node.submodules = collect_children_by_id(node, 'submodule') + node.imports = collect_children_by_prop(node, 'import', 'prefix') node.includes = collect_children(node, 'include') node.organization = maybe_child_property(node, 'organization', 'value') node.contact = maybe_child_property(node, 'contact', 'value') @@ -396,12 +397,12 @@ local function init_notification(node) node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.data = collect_data_children(node) + node.body = collect_data_children(node) end local function init_output(node) node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.data = collect_data_children_at_least_1(node) + node.body = collect_data_children_at_least_1(node) end local function init_path(node) -- TODO: parse path string @@ -644,8 +645,8 @@ function selftest() assert(schema.description == "Module to test YANG schema lib") -- Check both modules exist. (Also need to check they've loaded) - assert(schema.imports["ietf-inet-types"]) - assert(schema.imports["ietf-yang-types"]) + assert(schema.imports["inet"].id == "ietf-inet-types") + assert(schema.imports["yang"].id == "ietf-yang-types") -- Check all revisions are accounted for. assert(schema.revisions[1].description == "Revision 1") @@ -660,21 +661,21 @@ function selftest() -- Check the groupings assert(schema.body["fruit"]) assert(schema.body["fruit"].description) - assert(schema.body["fruit"].data["name"]) - assert(schema.body["fruit"].data["name"].kind == "leaf") - assert(schema.body["fruit"].data["name"].type.id == "string") - assert(schema.body["fruit"].data["name"].mandatory == true) - assert(schema.body["fruit"].data["name"].description == "Name of fruit.") - assert(schema.body["fruit"].data["score"].type.id == "uint8") - assert(schema.body["fruit"].data["score"].mandatory == true) - assert(schema.body["fruit"].data["score"].type.range.value[1] == 0) - assert(schema.body["fruit"].data["score"].type.range.value[2] == 10) + assert(schema.body["fruit"].body["name"]) + assert(schema.body["fruit"].body["name"].kind == "leaf") + assert(schema.body["fruit"].body["name"].type.id == "string") + assert(schema.body["fruit"].body["name"].mandatory == true) + assert(schema.body["fruit"].body["name"].description == "Name of fruit.") + assert(schema.body["fruit"].body["score"].type.id == "uint8") + assert(schema.body["fruit"].body["score"].mandatory == true) + assert(schema.body["fruit"].body["score"].type.range.value[1] == 0) + assert(schema.body["fruit"].body["score"].type.range.value[2] == 10) -- Check the containers description (NOT the leaf called "description") assert(schema.body["fruit-bowl"].description == "Represents a fruit bowl") -- Check the container has a leaf called "description" - local desc = schema.body["fruit-bowl"].data['description'] + local desc = schema.body["fruit-bowl"].body['description'] assert(desc.type.id == "string") assert(desc.description == "About the bowl") From 017ee827f16116d29cf2691976e0cc6a64aa1e6e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 25 Oct 2016 15:25:47 +0200 Subject: [PATCH 048/631] Add base IETF yang schemas --- src/lib/yang/ietf-inet-types.yang | 461 ++++++++++++++++++++++++++++ src/lib/yang/ietf-yang-types.yang | 480 ++++++++++++++++++++++++++++++ 2 files changed, 941 insertions(+) create mode 100644 src/lib/yang/ietf-inet-types.yang create mode 100644 src/lib/yang/ietf-yang-types.yang diff --git a/src/lib/yang/ietf-inet-types.yang b/src/lib/yang/ietf-inet-types.yang new file mode 100644 index 0000000000..5388b03933 --- /dev/null +++ b/src/lib/yang/ietf-inet-types.yang @@ -0,0 +1,461 @@ +module ietf-inet-types { + + namespace "urn:ietf:params:xml:ns:yang:ietf-inet-types"; + prefix "inet"; + + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; + + contact + "WG Web: + WG List: + + WG Chair: David Kessens + + + WG Chair: Juergen Schoenwaelder + + + Editor: Juergen Schoenwaelder + "; + + description + "This module contains a collection of generally useful derived + YANG data types for Internet addresses and related things. + + Copyright (c) 2013 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Simplified BSD License + set forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 6991; see + the RFC itself for full legal notices."; + + revision 2013-07-15 { + description + "This revision adds the following new data types: + - ip-address-no-zone + - ipv4-address-no-zone + - ipv6-address-no-zone"; + reference + "RFC 6991: Common YANG Data Types"; + } + + revision 2010-09-24 { + description + "Initial revision."; + reference + "RFC 6021: Common YANG Data Types"; + } + + /*** collection of types related to protocol fields ***/ + + typedef ip-version { + type enumeration { + enum unknown { + value "0"; + description + "An unknown or unspecified version of the Internet + protocol."; + } + enum ipv4 { + value "1"; + description + "The IPv4 protocol as defined in RFC 791."; + } + enum ipv6 { + value "2"; + description + "The IPv6 protocol as defined in RFC 2460."; + } + } + description + "This value represents the version of the IP protocol. + + In the value set and its semantics, this type is equivalent + to the InetVersion textual convention of the SMIv2."; + reference + "RFC 791: Internet Protocol + RFC 2460: Internet Protocol, Version 6 (IPv6) Specification + RFC 4001: Textual Conventions for Internet Network Addresses"; + } + + typedef dscp { + type uint8 { + range "0..63"; + } + description + "The dscp type represents a Differentiated Services Code Point + that may be used for marking packets in a traffic stream. + In the value set and its semantics, this type is equivalent + to the Dscp textual convention of the SMIv2."; + reference + "RFC 3289: Management Information Base for the Differentiated + Services Architecture + RFC 2474: Definition of the Differentiated Services Field + (DS Field) in the IPv4 and IPv6 Headers + RFC 2780: IANA Allocation Guidelines For Values In + the Internet Protocol and Related Headers"; + } + + typedef ipv6-flow-label { + type uint32 { + range "0..1048575"; + } + description + "The ipv6-flow-label type represents the flow identifier or Flow + Label in an IPv6 packet header that may be used to + discriminate traffic flows. + + In the value set and its semantics, this type is equivalent + to the IPv6FlowLabel textual convention of the SMIv2."; + reference + "RFC 3595: Textual Conventions for IPv6 Flow Label + RFC 2460: Internet Protocol, Version 6 (IPv6) Specification"; + } + + typedef port-number { + type uint16 { + range "0..65535"; + } + description + "The port-number type represents a 16-bit port number of an + Internet transport-layer protocol such as UDP, TCP, DCCP, or + SCTP. Port numbers are assigned by IANA. A current list of + all assignments is available from . + + Note that the port number value zero is reserved by IANA. In + situations where the value zero does not make sense, it can + be excluded by subtyping the port-number type. + In the value set and its semantics, this type is equivalent + to the InetPortNumber textual convention of the SMIv2."; + reference + "RFC 768: User Datagram Protocol + RFC 793: Transmission Control Protocol + RFC 4960: Stream Control Transmission Protocol + RFC 4340: Datagram Congestion Control Protocol (DCCP) + RFC 4001: Textual Conventions for Internet Network Addresses"; + } + + /*** collection of types related to autonomous systems ***/ + + typedef as-number { + type uint32; + description + "The as-number type represents autonomous system numbers + which identify an Autonomous System (AS). An AS is a set + of routers under a single technical administration, using + an interior gateway protocol and common metrics to route + packets within the AS, and using an exterior gateway + protocol to route packets to other ASes. IANA maintains + the AS number space and has delegated large parts to the + regional registries. + + Autonomous system numbers were originally limited to 16 + bits. BGP extensions have enlarged the autonomous system + number space to 32 bits. This type therefore uses an uint32 + base type without a range restriction in order to support + a larger autonomous system number space. + + In the value set and its semantics, this type is equivalent + to the InetAutonomousSystemNumber textual convention of + the SMIv2."; + reference + "RFC 1930: Guidelines for creation, selection, and registration + of an Autonomous System (AS) + RFC 4271: A Border Gateway Protocol 4 (BGP-4) + RFC 4001: Textual Conventions for Internet Network Addresses + RFC 6793: BGP Support for Four-Octet Autonomous System (AS) + Number Space"; + } + + /*** collection of types related to IP addresses and hostnames ***/ + + typedef ip-address { + type union { + type inet:ipv4-address; + type inet:ipv6-address; + } + description + "The ip-address type represents an IP address and is IP + version neutral. The format of the textual representation + implies the IP version. This type supports scoped addresses + by allowing zone identifiers in the address format."; + reference + "RFC 4007: IPv6 Scoped Address Architecture"; + } + + typedef ipv4-address { + type string { + pattern + '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '(%[\p{N}\p{L}]+)?'; + } + description + "The ipv4-address type represents an IPv4 address in + dotted-quad notation. The IPv4 address may include a zone + index, separated by a % sign. + + The zone index is used to disambiguate identical address + values. For link-local addresses, the zone index will + typically be the interface index number or the name of an + interface. If the zone index is not present, the default + zone of the device will be used. + + The canonical format for the zone index is the numerical + format"; + } + + typedef ipv6-address { + type string { + pattern '((:|[0-9a-fA-F]{0,4}):)([0-9a-fA-F]{0,4}:){0,5}' + + '((([0-9a-fA-F]{0,4}:)?(:|[0-9a-fA-F]{0,4}))|' + + '(((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}' + + '(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])))' + + '(%[\p{N}\p{L}]+)?'; + pattern '(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|' + + '((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)' + + '(%.+)?'; + } + description + "The ipv6-address type represents an IPv6 address in full, + mixed, shortened, and shortened-mixed notation. The IPv6 + address may include a zone index, separated by a % sign. + + The zone index is used to disambiguate identical address + values. For link-local addresses, the zone index will + typically be the interface index number or the name of an + interface. If the zone index is not present, the default + zone of the device will be used. + + + + The canonical format of IPv6 addresses uses the textual + representation defined in Section 4 of RFC 5952. The + canonical format for the zone index is the numerical + format as described in Section 11.2 of RFC 4007."; + reference + "RFC 4291: IP Version 6 Addressing Architecture + RFC 4007: IPv6 Scoped Address Architecture + RFC 5952: A Recommendation for IPv6 Address Text + Representation"; + } + + typedef ip-address-no-zone { + type union { + type inet:ipv4-address-no-zone; + type inet:ipv6-address-no-zone; + } + description + "The ip-address-no-zone type represents an IP address and is + IP version neutral. The format of the textual representation + implies the IP version. This type does not support scoped + addresses since it does not allow zone identifiers in the + address format."; + reference + "RFC 4007: IPv6 Scoped Address Architecture"; + } + + typedef ipv4-address-no-zone { + type inet:ipv4-address { + pattern '[0-9\.]*'; + } + description + "An IPv4 address without a zone index. This type, derived from + ipv4-address, may be used in situations where the zone is + known from the context and hence no zone index is needed."; + } + + typedef ipv6-address-no-zone { + type inet:ipv6-address { + pattern '[0-9a-fA-F:\.]*'; + } + description + "An IPv6 address without a zone index. This type, derived from + ipv6-address, may be used in situations where the zone is + known from the context and hence no zone index is needed."; + reference + "RFC 4291: IP Version 6 Addressing Architecture + RFC 4007: IPv6 Scoped Address Architecture + RFC 5952: A Recommendation for IPv6 Address Text + Representation"; + } + + typedef ip-prefix { + type union { + type inet:ipv4-prefix; + type inet:ipv6-prefix; + } + description + "The ip-prefix type represents an IP prefix and is IP + version neutral. The format of the textual representations + implies the IP version."; + } + + typedef ipv4-prefix { + type string { + pattern + '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])' + + '/(([0-9])|([1-2][0-9])|(3[0-2]))'; + } + description + "The ipv4-prefix type represents an IPv4 address prefix. + The prefix length is given by the number following the + slash character and must be less than or equal to 32. + + A prefix length value of n corresponds to an IP address + mask that has n contiguous 1-bits from the most + significant bit (MSB) and all other bits set to 0. + + The canonical format of an IPv4 prefix has all bits of + the IPv4 address set to zero that are not part of the + IPv4 prefix."; + } + + typedef ipv6-prefix { + type string { + pattern '((:|[0-9a-fA-F]{0,4}):)([0-9a-fA-F]{0,4}:){0,5}' + + '((([0-9a-fA-F]{0,4}:)?(:|[0-9a-fA-F]{0,4}))|' + + '(((25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])\.){3}' + + '(25[0-5]|2[0-4][0-9]|[01]?[0-9]?[0-9])))' + + '(/(([0-9])|([0-9]{2})|(1[0-1][0-9])|(12[0-8])))'; + pattern '(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|' + + '((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)' + + '(/.+)'; + } + + + description + "The ipv6-prefix type represents an IPv6 address prefix. + The prefix length is given by the number following the + slash character and must be less than or equal to 128. + + A prefix length value of n corresponds to an IP address + mask that has n contiguous 1-bits from the most + significant bit (MSB) and all other bits set to 0. + + The IPv6 address should have all bits that do not belong + to the prefix set to zero. + + The canonical format of an IPv6 prefix has all bits of + the IPv6 address set to zero that are not part of the + IPv6 prefix. Furthermore, the IPv6 address is represented + as defined in Section 4 of RFC 5952."; + reference + "RFC 5952: A Recommendation for IPv6 Address Text + Representation"; + } + + /*** collection of domain name and URI types ***/ + + typedef domain-name { + type string { + pattern + '((([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.)*' + + '([a-zA-Z0-9_]([a-zA-Z0-9\-_]){0,61})?[a-zA-Z0-9]\.?)' + + '|\.'; + length "1..253"; + } + description + "The domain-name type represents a DNS domain name. The + name SHOULD be fully qualified whenever possible. + + Internet domain names are only loosely specified. Section + 3.5 of RFC 1034 recommends a syntax (modified in Section + 2.1 of RFC 1123). The pattern above is intended to allow + for current practice in domain name use, and some possible + future expansion. It is designed to hold various types of + domain names, including names used for A or AAAA records + (host names) and other records, such as SRV records. Note + that Internet host names have a stricter syntax (described + in RFC 952) than the DNS recommendations in RFCs 1034 and + 1123, and that systems that want to store host names in + schema nodes using the domain-name type are recommended to + adhere to this stricter standard to ensure interoperability. + + The encoding of DNS names in the DNS protocol is limited + to 255 characters. Since the encoding consists of labels + prefixed by a length bytes and there is a trailing NULL + byte, only 253 characters can appear in the textual dotted + notation. + + The description clause of schema nodes using the domain-name + type MUST describe when and how these names are resolved to + IP addresses. Note that the resolution of a domain-name value + may require to query multiple DNS records (e.g., A for IPv4 + and AAAA for IPv6). The order of the resolution process and + which DNS record takes precedence can either be defined + explicitly or may depend on the configuration of the + resolver. + + Domain-name values use the US-ASCII encoding. Their canonical + format uses lowercase US-ASCII characters. Internationalized + domain names MUST be A-labels as per RFC 5890."; + reference + "RFC 952: DoD Internet Host Table Specification + RFC 1034: Domain Names - Concepts and Facilities + RFC 1123: Requirements for Internet Hosts -- Application + and Support + RFC 2782: A DNS RR for specifying the location of services + (DNS SRV) + RFC 5890: Internationalized Domain Names in Applications + (IDNA): Definitions and Document Framework"; + } + + typedef host { + type union { + type inet:ip-address; + type inet:domain-name; + } + description + "The host type represents either an IP address or a DNS + domain name."; + } + + typedef uri { + type string; + description + "The uri type represents a Uniform Resource Identifier + (URI) as defined by STD 66. + + Objects using the uri type MUST be in US-ASCII encoding, + and MUST be normalized as described by RFC 3986 Sections + 6.2.1, 6.2.2.1, and 6.2.2.2. All unnecessary + percent-encoding is removed, and all case-insensitive + characters are set to lowercase except for hexadecimal + digits, which are normalized to uppercase as described in + Section 6.2.2.1. + + The purpose of this normalization is to help provide + unique URIs. Note that this normalization is not + sufficient to provide uniqueness. Two URIs that are + textually distinct after this normalization may still be + equivalent. + + Objects using the uri type may restrict the schemes that + they permit. For example, 'data:' and 'urn:' schemes + might not be appropriate. + + A zero-length URI is not a valid URI. This can be used to + express 'URI absent' where required. + + In the value set and its semantics, this type is equivalent + to the Uri SMIv2 textual convention defined in RFC 5017."; + reference + "RFC 3986: Uniform Resource Identifier (URI): Generic Syntax + RFC 3305: Report from the Joint W3C/IETF URI Planning Interest + Group: Uniform Resource Identifiers (URIs), URLs, + and Uniform Resource Names (URNs): Clarifications + and Recommendations + RFC 5017: MIB Textual Conventions for Uniform Resource + Identifiers (URIs)"; + } + +} diff --git a/src/lib/yang/ietf-yang-types.yang b/src/lib/yang/ietf-yang-types.yang new file mode 100644 index 0000000000..371a091d14 --- /dev/null +++ b/src/lib/yang/ietf-yang-types.yang @@ -0,0 +1,480 @@ +module ietf-yang-types { + + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-types"; + prefix "yang"; + + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; + + contact + "WG Web: + WG List: + + WG Chair: David Kessens + + + WG Chair: Juergen Schoenwaelder + + + Editor: Juergen Schoenwaelder + "; + + description + "This module contains a collection of generally useful derived + YANG data types. + + Copyright (c) 2013 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject + to the license terms contained in, the Simplified BSD License + set forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 6991; see + the RFC itself for full legal notices."; + + revision 2013-07-15 { + description + "This revision adds the following new data types: + - yang-identifier + - hex-string + - uuid + - dotted-quad"; + reference + "RFC 6991: Common YANG Data Types"; + } + + revision 2010-09-24 { + description + "Initial revision."; + reference + "RFC 6021: Common YANG Data Types"; + } + + /*** collection of counter and gauge types ***/ + + typedef counter32 { + type uint32; + description + "The counter32 type represents a non-negative integer + that monotonically increases until it reaches a + maximum value of 2^32-1 (4294967295 decimal), when it + wraps around and starts increasing again from zero. + + Counters have no defined 'initial' value, and thus, a + single value of a counter has (in general) no information + content. Discontinuities in the monotonically increasing + value normally occur at re-initialization of the + management system, and at other times as specified in the + description of a schema node using this type. If such + other times can occur, for example, the creation of + a schema node of type counter32 at times other than + re-initialization, then a corresponding schema node + should be defined, with an appropriate type, to indicate + the last discontinuity. + + The counter32 type should not be used for configuration + schema nodes. A default statement SHOULD NOT be used in + combination with the type counter32. + + In the value set and its semantics, this type is equivalent + to the Counter32 type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef zero-based-counter32 { + type yang:counter32; + default "0"; + description + "The zero-based-counter32 type represents a counter32 + that has the defined 'initial' value zero. + + A schema node of this type will be set to zero (0) on creation + and will thereafter increase monotonically until it reaches + a maximum value of 2^32-1 (4294967295 decimal), when it + wraps around and starts increasing again from zero. + + Provided that an application discovers a new schema node + of this type within the minimum time to wrap, it can use the + 'initial' value as a delta. It is important for a management + station to be aware of this minimum time and the actual time + between polls, and to discard data if the actual time is too + long or there is no defined minimum time. + + In the value set and its semantics, this type is equivalent + to the ZeroBasedCounter32 textual convention of the SMIv2."; + reference + "RFC 4502: Remote Network Monitoring Management Information + Base Version 2"; + } + + typedef counter64 { + type uint64; + description + "The counter64 type represents a non-negative integer + that monotonically increases until it reaches a + maximum value of 2^64-1 (18446744073709551615 decimal), + when it wraps around and starts increasing again from zero. + + Counters have no defined 'initial' value, and thus, a + single value of a counter has (in general) no information + content. Discontinuities in the monotonically increasing + value normally occur at re-initialization of the + management system, and at other times as specified in the + description of a schema node using this type. If such + other times can occur, for example, the creation of + a schema node of type counter64 at times other than + re-initialization, then a corresponding schema node + should be defined, with an appropriate type, to indicate + the last discontinuity. + + The counter64 type should not be used for configuration + schema nodes. A default statement SHOULD NOT be used in + combination with the type counter64. + + In the value set and its semantics, this type is equivalent + to the Counter64 type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef zero-based-counter64 { + type yang:counter64; + default "0"; + description + "The zero-based-counter64 type represents a counter64 that + has the defined 'initial' value zero. + + + + + A schema node of this type will be set to zero (0) on creation + and will thereafter increase monotonically until it reaches + a maximum value of 2^64-1 (18446744073709551615 decimal), + when it wraps around and starts increasing again from zero. + + Provided that an application discovers a new schema node + of this type within the minimum time to wrap, it can use the + 'initial' value as a delta. It is important for a management + station to be aware of this minimum time and the actual time + between polls, and to discard data if the actual time is too + long or there is no defined minimum time. + + In the value set and its semantics, this type is equivalent + to the ZeroBasedCounter64 textual convention of the SMIv2."; + reference + "RFC 2856: Textual Conventions for Additional High Capacity + Data Types"; + } + + typedef gauge32 { + type uint32; + description + "The gauge32 type represents a non-negative integer, which + may increase or decrease, but shall never exceed a maximum + value, nor fall below a minimum value. The maximum value + cannot be greater than 2^32-1 (4294967295 decimal), and + the minimum value cannot be smaller than 0. The value of + a gauge32 has its maximum value whenever the information + being modeled is greater than or equal to its maximum + value, and has its minimum value whenever the information + being modeled is smaller than or equal to its minimum value. + If the information being modeled subsequently decreases + below (increases above) the maximum (minimum) value, the + gauge32 also decreases (increases). + + In the value set and its semantics, this type is equivalent + to the Gauge32 type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef gauge64 { + type uint64; + description + "The gauge64 type represents a non-negative integer, which + may increase or decrease, but shall never exceed a maximum + value, nor fall below a minimum value. The maximum value + cannot be greater than 2^64-1 (18446744073709551615), and + the minimum value cannot be smaller than 0. The value of + a gauge64 has its maximum value whenever the information + being modeled is greater than or equal to its maximum + value, and has its minimum value whenever the information + being modeled is smaller than or equal to its minimum value. + If the information being modeled subsequently decreases + below (increases above) the maximum (minimum) value, the + gauge64 also decreases (increases). + + In the value set and its semantics, this type is equivalent + to the CounterBasedGauge64 SMIv2 textual convention defined + in RFC 2856"; + reference + "RFC 2856: Textual Conventions for Additional High Capacity + Data Types"; + } + + /*** collection of identifier-related types ***/ + + typedef object-identifier { + type string { + pattern '(([0-1](\.[1-3]?[0-9]))|(2\.(0|([1-9]\d*))))' + + '(\.(0|([1-9]\d*)))*'; + } + description + "The object-identifier type represents administratively + assigned names in a registration-hierarchical-name tree. + + Values of this type are denoted as a sequence of numerical + non-negative sub-identifier values. Each sub-identifier + value MUST NOT exceed 2^32-1 (4294967295). Sub-identifiers + are separated by single dots and without any intermediate + whitespace. + + The ASN.1 standard restricts the value space of the first + sub-identifier to 0, 1, or 2. Furthermore, the value space + of the second sub-identifier is restricted to the range + 0 to 39 if the first sub-identifier is 0 or 1. Finally, + the ASN.1 standard requires that an object identifier + has always at least two sub-identifiers. The pattern + captures these restrictions. + + Although the number of sub-identifiers is not limited, + module designers should realize that there may be + implementations that stick with the SMIv2 limit of 128 + sub-identifiers. + + This type is a superset of the SMIv2 OBJECT IDENTIFIER type + since it is not restricted to 128 sub-identifiers. Hence, + this type SHOULD NOT be used to represent the SMIv2 OBJECT + IDENTIFIER type; the object-identifier-128 type SHOULD be + used instead."; + reference + "ISO9834-1: Information technology -- Open Systems + Interconnection -- Procedures for the operation of OSI + Registration Authorities: General procedures and top + arcs of the ASN.1 Object Identifier tree"; + } + + typedef object-identifier-128 { + type object-identifier { + pattern '\d*(\.\d*){1,127}'; + } + description + "This type represents object-identifiers restricted to 128 + sub-identifiers. + + In the value set and its semantics, this type is equivalent + to the OBJECT IDENTIFIER type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef yang-identifier { + type string { + length "1..max"; + pattern '[a-zA-Z_][a-zA-Z0-9\-_.]*'; + pattern '.|..|[^xX].*|.[^mM].*|..[^lL].*'; + } + description + "A YANG identifier string as defined by the 'identifier' + rule in Section 12 of RFC 6020. An identifier must + start with an alphabetic character or an underscore + followed by an arbitrary sequence of alphabetic or + numeric characters, underscores, hyphens, or dots. + + A YANG identifier MUST NOT start with any possible + combination of the lowercase or uppercase character + sequence 'xml'."; + reference + "RFC 6020: YANG - A Data Modeling Language for the Network + Configuration Protocol (NETCONF)"; + } + + /*** collection of types related to date and time***/ + + typedef date-and-time { + type string { + pattern '\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?' + + '(Z|[\+\-]\d{2}:\d{2})'; + } + description + "The date-and-time type is a profile of the ISO 8601 + standard for representation of dates and times using the + Gregorian calendar. The profile is defined by the + date-time production in Section 5.6 of RFC 3339. + + The date-and-time type is compatible with the dateTime XML + schema type with the following notable exceptions: + + (a) The date-and-time type does not allow negative years. + + (b) The date-and-time time-offset -00:00 indicates an unknown + time zone (see RFC 3339) while -00:00 and +00:00 and Z + all represent the same time zone in dateTime. + + (c) The canonical format (see below) of data-and-time values + differs from the canonical format used by the dateTime XML + schema type, which requires all times to be in UTC using + the time-offset 'Z'. + + This type is not equivalent to the DateAndTime textual + convention of the SMIv2 since RFC 3339 uses a different + separator between full-date and full-time and provides + higher resolution of time-secfrac. + + The canonical format for date-and-time values with a known time + zone uses a numeric time zone offset that is calculated using + the device's configured known offset to UTC time. A change of + the device's offset to UTC time will cause date-and-time values + to change accordingly. Such changes might happen periodically + in case a server follows automatically daylight saving time + (DST) time zone offset changes. The canonical format for + date-and-time values with an unknown time zone (usually + referring to the notion of local time) uses the time-offset + -00:00."; + reference + "RFC 3339: Date and Time on the Internet: Timestamps + RFC 2579: Textual Conventions for SMIv2 + XSD-TYPES: XML Schema Part 2: Datatypes Second Edition"; + } + + typedef timeticks { + type uint32; + description + "The timeticks type represents a non-negative integer that + represents the time, modulo 2^32 (4294967296 decimal), in + hundredths of a second between two epochs. When a schema + node is defined that uses this type, the description of + the schema node identifies both of the reference epochs. + + In the value set and its semantics, this type is equivalent + to the TimeTicks type of the SMIv2."; + reference + "RFC 2578: Structure of Management Information Version 2 + (SMIv2)"; + } + + typedef timestamp { + type yang:timeticks; + description + "The timestamp type represents the value of an associated + timeticks schema node at which a specific occurrence + happened. The specific occurrence must be defined in the + description of any schema node defined using this type. When + the specific occurrence occurred prior to the last time the + associated timeticks attribute was zero, then the timestamp + value is zero. Note that this requires all timestamp values + to be reset to zero when the value of the associated timeticks + attribute reaches 497+ days and wraps around to zero. + + The associated timeticks schema node must be specified + in the description of any schema node using this type. + + In the value set and its semantics, this type is equivalent + to the TimeStamp textual convention of the SMIv2."; + reference + "RFC 2579: Textual Conventions for SMIv2"; + } + + /*** collection of generic address types ***/ + + typedef phys-address { + type string { + pattern '([0-9a-fA-F]{2}(:[0-9a-fA-F]{2})*)?'; + } + + + + + description + "Represents media- or physical-level addresses represented + as a sequence octets, each octet represented by two hexadecimal + numbers. Octets are separated by colons. The canonical + representation uses lowercase characters. + + In the value set and its semantics, this type is equivalent + to the PhysAddress textual convention of the SMIv2."; + reference + "RFC 2579: Textual Conventions for SMIv2"; + } + + typedef mac-address { + type string { + pattern '[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}'; + } + description + "The mac-address type represents an IEEE 802 MAC address. + The canonical representation uses lowercase characters. + + In the value set and its semantics, this type is equivalent + to the MacAddress textual convention of the SMIv2."; + reference + "IEEE 802: IEEE Standard for Local and Metropolitan Area + Networks: Overview and Architecture + RFC 2579: Textual Conventions for SMIv2"; + } + + /*** collection of XML-specific types ***/ + + typedef xpath1.0 { + type string; + description + "This type represents an XPATH 1.0 expression. + + When a schema node is defined that uses this type, the + description of the schema node MUST specify the XPath + context in which the XPath expression is evaluated."; + reference + "XPATH: XML Path Language (XPath) Version 1.0"; + } + + /*** collection of string types ***/ + + typedef hex-string { + type string { + pattern '([0-9a-fA-F]{2}(:[0-9a-fA-F]{2})*)?'; + } + description + "A hexadecimal string with octets represented as hex digits + separated by colons. The canonical representation uses + lowercase characters."; + } + + typedef uuid { + type string { + pattern '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-' + + '[0-9a-fA-F]{4}-[0-9a-fA-F]{12}'; + } + description + "A Universally Unique IDentifier in the string representation + defined in RFC 4122. The canonical representation uses + lowercase characters. + + The following is an example of a UUID in string representation: + f81d4fae-7dec-11d0-a765-00a0c91e6bf6 + "; + reference + "RFC 4122: A Universally Unique IDentifier (UUID) URN + Namespace"; + } + + typedef dotted-quad { + type string { + pattern + '(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.){3}' + + '([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])'; + } + description + "An unsigned 32-bit number expressed in the dotted-quad + notation, i.e., four octets written as decimal numbers + and separated with the '.' (full stop) character."; + } +} From edf611d71219c169120dba0ea3fc625ebf9bde90 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 25 Oct 2016 15:09:33 +0200 Subject: [PATCH 049/631] Add linking/partial evaluation phase to yang * src/lib/yang/schema2.lua: Modules just collect "body" children; extensions, features etc are collected separately. Resolve submodules and module imports. Resolve if-feature and link groupings into uses. Resolve all "when" expressions as true. Add load_schema_by_name facility, and change tests to check resolved output. --- src/lib/yang/schema2.lua | 331 +++++++++++++++++++++++++++++---------- 1 file changed, 248 insertions(+), 83 deletions(-) diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua index 05fb3c11e1..edb1272fa3 100644 --- a/src/lib/yang/schema2.lua +++ b/src/lib/yang/schema2.lua @@ -115,14 +115,6 @@ local function collect_children_by_id(node, kinds) end local function collect_body_children(node) - return collect_children_by_id( - node, - {'extension', 'feature', 'identity', 'typedef', 'grouping', - 'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', - 'anyxml', 'rpc', 'notification', 'deviation'}) -end - -local function collect_data_children(node) return collect_children_by_id( node, {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml'}) @@ -133,8 +125,8 @@ local function at_least_one(tab) return false end -local function collect_data_children_at_least_1(node) - local ret = collect_data_children(node) +local function collect_body_children_at_least_1(node) + local ret = collect_body_children(node) if not at_least_one(ret) then error_with_path(node.path, "missing data statements") end @@ -209,7 +201,7 @@ end local function init_anyxml(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.must = collect_child_properties(node, 'must', 'value') node.config = maybe_child_property(node, 'config', 'value') node.mandatory = maybe_child_property(node, 'mandatory', 'value') @@ -224,7 +216,7 @@ end local function init_augment(node) node.node_id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -237,16 +229,16 @@ end local function init_case(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.body = collect_data_children(node) + node.body = collect_body_children(node) end local function init_choice(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.default = maybe_child_property(node, 'default', 'value') node.config = maybe_child_property(node, 'config', 'value') node.mandatory = maybe_child_property(node, 'mandatory', 'value') @@ -262,7 +254,7 @@ end local function init_container(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.must = collect_child_properties(node, 'must', 'value') node.presence = maybe_child_property(node, 'presence', 'value') node.config = maybe_child_property(node, 'config', 'value') @@ -271,7 +263,7 @@ local function init_container(node) node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_data_children(node) + node.body = collect_body_children(node) end local function init_extension(node) node.id = require_argument(node) @@ -282,7 +274,7 @@ local function init_extension(node) end local function init_feature(node) node.id = require_argument(node) - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -294,7 +286,7 @@ local function init_grouping(node) node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_data_children(node) + node.body = collect_body_children(node) end local function init_identity(node) node.id = require_argument(node) @@ -306,6 +298,7 @@ end local function init_import(node) node.id = require_argument(node) node.prefix = require_child_property(node, 'prefix', 'value') + node.revision_date = maybe_child_property(node, 'revision-date', 'value') end local function init_include(node) node.id = require_argument(node) @@ -314,12 +307,12 @@ end local function init_input(node) node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_data_children_at_least_1(node) + node.body = collect_body_children_at_least_1(node) end local function init_leaf(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.type = require_child(node, 'type') node.units = maybe_child_property(node, 'units', 'value') node.must = collect_child_properties(node, 'must', 'value') @@ -333,7 +326,7 @@ end local function init_leaf_list(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.type = require_child(node, 'type') node.units = maybe_child_property(node, 'units', 'value') node.must = collect_child_properties(node, 'must', 'value') @@ -354,7 +347,7 @@ end local function init_list(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.must = collect_child_properties(node, 'must', 'value') node.key = maybe_child_property(node, 'key', 'value') node.unique = collect_child_properties(node, 'unique', 'value') @@ -365,7 +358,7 @@ local function init_list(node) node.status = maybe_child_property(node, 'status', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_data_children_at_least_1(node) + node.body = collect_body_children_at_least_1(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end @@ -374,15 +367,22 @@ local function init_module(node) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.namespace = require_child_property(node, 'namespace', 'value') node.prefix = require_child_property(node, 'prefix', 'value') - node.submodules = collect_children_by_id(node, 'submodule') node.imports = collect_children_by_prop(node, 'import', 'prefix') - node.includes = collect_children(node, 'include') + node.includes = collect_children_by_id(node, 'include') node.organization = maybe_child_property(node, 'organization', 'value') node.contact = maybe_child_property(node, 'contact', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') node.revisions = collect_children(node, 'revision') node.augments = collect_children(node, 'augment') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.features = collect_children_by_id(node, 'feature') + node.extensions = collect_children_by_id(node, 'extension') + node.identities = collect_children_by_id(node, 'identity') + node.rpcs = collect_children_by_id(node, 'rpc') + node.notifications = collect_children_by_id(node, 'notification') + node.deviations = collect_children_by_id(node, 'deviation') node.body = collect_body_children(node) end local function init_namespace(node) @@ -391,18 +391,18 @@ local function init_namespace(node) end local function init_notification(node) node.id = require_argument(node) - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_data_children(node) + node.body = collect_body_children(node) end local function init_output(node) node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_data_children_at_least_1(node) + node.body = collect_body_children_at_least_1(node) end local function init_path(node) -- TODO: parse path string @@ -443,7 +443,7 @@ local function init_revision(node) end local function init_rpc(node) node.id = require_argument(node) - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -478,6 +478,14 @@ local function init_submodule(node) node.reference = maybe_child_property(node, 'reference', 'value') node.revisions = collect_children(node, 'revision') node.augments = collect_children(node, 'augment') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.features = collect_children_by_id(node, 'feature') + node.extensions = collect_children_by_id(node, 'extension') + node.identities = collect_children_by_id(node, 'identity') + node.rpcs = collect_children_by_id(node, 'rpc') + node.notifications = collect_children_by_id(node, 'notification') + node.deviations = collect_children_by_id(node, 'deviation') node.body = collect_body_children(node) end local function init_typedef(node) @@ -492,7 +500,7 @@ end local function init_uses(node) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_feature = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -554,20 +562,176 @@ declare_initializer(init_uses, 'uses') declare_initializer(init_value, 'value') local function schema_from_ast(ast) - assert(#ast == 1, 'expected a single module form') - local parsed = parse_node(ast[1]) - assert(parsed.kind == 'module', 'not a yang module') - return parsed + local ret + local submodules = {} + for _,node in ipairs(ast) do + if node.keyword == 'module' then + assert(not ret, 'expected only one module form') + ret = parse_node(node) + elseif node.keyword == 'submodule' then + assert(not submodules[node.id], 'duplicate submodule name: '..node.id) + submodules[node.id] = parse_node(node) + else + error('expected only module and submodule statements, got: '..node.keyword) + end + end + assert(ret, 'missing module form') + ret.submodules = submodules + return ret +end + +-- Strip properties pertaining to original source representation. +local function strip(exp) + if type(exp) ~= 'table' then return exp end + local ret = {} + for k, v in pairs(exp) do + if k ~= 'children' and k ~= 'argument_string' and k ~= 'order' and k ~= 'path' then + ret[k] = strip(v) + end + end + return ret +end + +-- Inline "grouping" into "uses". +-- Inline "submodule" into "include". +-- Inline "imports" into "module". +-- Inline "typedef" into "type". (TODO) +-- Resolve if-feature. +-- Warn on any "when", resolving them as being true. +-- Resolve all augment and refine nodes. (TODO) +function resolve(schema, features) + local function shallow_copy(node) + local out = {} + for k,v in pairs(node) do out[k] = v end + return out + end + local function pop_prop(node, prop) + local val = node[prop] + node[prop] = nil + return val + end + local function push_env(node, env) + node = shallow_copy(node) + return node, {env=env, + groupings=pop_prop(node, 'groupings'), + typedefs=pop_prop(node, 'typedefs')} + end + local function lookup(env, prop, name) + if not env then error(prop..' not found: '..name) end + if not env[prop] or not env[prop][name] then + return lookup(env.env, prop, name) + end + return env[prop][name], env + end + local visit + local function instantiate_grouping(env, name) + local grouping, grouping_env = lookup(env, 'groupings', name) + return visit(grouping, grouping_env) + end + function visit(node, env) + node, env = push_env(node, env) + local when = pop_prop(node, 'when') + if when then + print('warning: assuming "when" condition to be true: '..when.value) + end + for k,v in pairs(pop_prop(node, 'features') or {}) do + v = visit(v, env) + if v then + if not env.features then env.features = {} end + env.features[k] = v + end + end + for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do + if not has_feature(env, feature) then return nil, env end + end + for k,v in pairs(node.body or {}) do + if v.kind == 'uses' then + -- Inline "grouping" into "uses". + local grouping = instantiate_grouping(env, v.id) + node.body[k] = nil + for k,v in pairs(grouping.body) do + assert(not node.body[k], 'duplicate identifier: '..k) + node.body[k] = v + end + -- TODO: Handle refine and augment statements. + else + node.body[k] = visit(v, env) + end + end + return node, env + end + local function merge_tables(dst, src) + for k,v in pairs(src) do + assert(dst[k] == nil or dst[k] == v, 'incompatible definitions: '..k) + dst[k] = v + end + end + local linked = {} + local function link(node, env) + if linked[node.id] then + assert(linked[node.id] ~= 'pending', 'recursive import of '..node.id) + local node, env = unpack(linked[node.id]) + return node, env + end + linked[node.id] = 'pending' + node = shallow_copy(node) + local module_env = {env=env, prefixes={}, extensions={}, features={}, + identities={}, typedefs={}, groupings={}} + local module_body = shallow_copy(node.body) + for k,v in pairs(pop_prop(node, 'includes')) do + local submodule = lookup(env, 'submodules', k) + assert(submodule.belongs_to.id == node.id) + submodule = link(submodule, env) + merge_tables(module_env.extensions, submodule.env.extensions) + merge_tables(module_env.features, submodule.env.features) + merge_tables(module_env.identities, submodule.env.identities) + merge_tables(module_env.typedefs, submodule.env.typedefs) + merge_tables(module_env.groupings, submodule.env.groupings) + merge_tables(module_body, submodule.body) + end + if node.prefix then + assert(node.kind == 'module', node.kind) + module_env.prefixes[node.prefix] = module_env + end + for k,v in pairs(pop_prop(node, 'imports')) do + assert(not module_env.prefixes[v.prefix], 'duplicate prefix') + -- CHECKME: Discarding body from import, just importing env. + -- Is this OK? + local schema, env = load_schema_by_name(v.id, v.revision_date) + module_env.prefixes[v.prefix] = env + end + node, env = visit(node, module_env) + -- The typedefs, groupings, identities, and so on of this module + -- are externally visible for other modules that may import this + -- one; save them and their environment. + linked[node.id] = {node, env} + return node, env + end + schema = shallow_copy(schema) + return link(schema, {features=(features or {}), + submodules=pop_prop(schema, 'submodules')}) end function parse_schema(src, filename) return schema_from_ast(parser.parse_string(src, filename)) end - function parse_schema_file(filename) return schema_from_ast(parser.parse_file(filename)) end +function load_schema(src, filename) + return resolve(strip(parse_schema(src, filename))) +end +function load_schema_file(filename) + return resolve(strip(parse_schema_file(filename))) +end +function load_schema_by_name(name, revision) + -- FIXME: @ is not valid in a Lua module name. + -- if revision then name = name .. '@' .. revision end + name = name:gsub('-', '_') + return load_schema(require('lib.yang.'..name..'_yang'), name) +end + function selftest() local test_schema = [[module fruit { namespace "urn:testing:fruit"; @@ -598,7 +762,7 @@ function selftest() } grouping fruit { - description "Represets a piece of fruit"; + description "Represents a piece of fruit"; leaf name { type string; @@ -634,50 +798,51 @@ function selftest() } }]] - -- Convert the schema using the already tested parser. - local schema = parse_schema(test_schema, "schema selftest") - - assert(schema.id == "fruit") - assert(schema.namespace == "urn:testing:fruit") - assert(schema.prefix == "fruit") - assert(schema.contact == "John Smith fake@person.tld") - assert(schema.organization == "Fruit Inc.") - assert(schema.description == "Module to test YANG schema lib") - - -- Check both modules exist. (Also need to check they've loaded) - assert(schema.imports["inet"].id == "ietf-inet-types") - assert(schema.imports["yang"].id == "ietf-yang-types") - - -- Check all revisions are accounted for. - assert(schema.revisions[1].description == "Revision 1") - assert(schema.revisions[1].value == "2016-05-27") - assert(schema.revisions[2].description == "Revision 2") - assert(schema.revisions[2].value == "2016-05-28") - - -- Check that the feature statements are there. - assert(schema.body["bowl"]) - assert(schema.body["bowl"].kind == 'feature') - - -- Check the groupings - assert(schema.body["fruit"]) - assert(schema.body["fruit"].description) - assert(schema.body["fruit"].body["name"]) - assert(schema.body["fruit"].body["name"].kind == "leaf") - assert(schema.body["fruit"].body["name"].type.id == "string") - assert(schema.body["fruit"].body["name"].mandatory == true) - assert(schema.body["fruit"].body["name"].description == "Name of fruit.") - assert(schema.body["fruit"].body["score"].type.id == "uint8") - assert(schema.body["fruit"].body["score"].mandatory == true) - assert(schema.body["fruit"].body["score"].type.range.value[1] == 0) - assert(schema.body["fruit"].body["score"].type.range.value[2] == 10) - - -- Check the containers description (NOT the leaf called "description") - assert(schema.body["fruit-bowl"].description == "Represents a fruit bowl") - - -- Check the container has a leaf called "description" - local desc = schema.body["fruit-bowl"].body['description'] - assert(desc.type.id == "string") - assert(desc.description == "About the bowl") - - parse_schema(require('lib.yang.example_yang')) + local schema, env = load_schema(test_schema) + assert(schema.id == "fruit") + assert(schema.namespace == "urn:testing:fruit") + assert(schema.prefix == "fruit") + assert(schema.contact == "John Smith fake@person.tld") + assert(schema.organization == "Fruit Inc.") + assert(schema.description == "Module to test YANG schema lib") + + -- Check all revisions are accounted for. + assert(schema.revisions[1].description == "Revision 1") + assert(schema.revisions[1].value == "2016-05-27") + assert(schema.revisions[2].description == "Revision 2") + assert(schema.revisions[2].value == "2016-05-28") + + -- Check that the feature statements are in the exports interface + -- but not the schema itself. + assert(not schema.features) + assert(env.features["bowl"]) + assert(env.features["bowl"].description == 'A fruit bowl') + + -- Check that groupings get inlined into their uses. + assert(schema.body['fruit-bowl']) + assert(schema.body['fruit-bowl'].description == 'Represents a fruit bowl') + local contents = schema.body['fruit-bowl'].body['contents'] + assert(contents) + assert(contents.kind == 'list') + -- TODO: Copy description over? Probably not given that one node + -- can use multiple groupings. + -- assert(contents.description == 'Represents a piece of fruit') + assert(contents.body['name'].kind == 'leaf') + assert(contents.body['name'].type.id == 'string') + assert(contents.body["name"].mandatory == true) + assert(contents.body["name"].description == "Name of fruit.") + assert(contents.body["score"].type.id == "uint8") + assert(contents.body["score"].mandatory == true) + assert(contents.body["score"].type.range.value[1] == 0) + assert(contents.body["score"].type.range.value[2] == 10) + + -- Check the container has a leaf called "description" + local desc = schema.body["fruit-bowl"].body['description'] + assert(desc.type.id == "string") + assert(desc.description == "About the bowl") + + parse_schema(require('lib.yang.ietf_yang_types_yang')) + parse_schema(require('lib.yang.ietf_inet_types_yang')) + + load_schema_by_name('ietf-yang-types') end From 54a4f9abf2633d331c9e9a0e06cca5242c40634b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 25 Oct 2016 15:11:35 +0200 Subject: [PATCH 050/631] Remove example.yang * src/lib/yang/example.yang: Removed, as it uses modules that we don't include. --- src/lib/yang/example.yang | 1188 ------------------------------------- 1 file changed, 1188 deletions(-) delete mode 100644 src/lib/yang/example.yang diff --git a/src/lib/yang/example.yang b/src/lib/yang/example.yang deleted file mode 100644 index aab5041ac6..0000000000 --- a/src/lib/yang/example.yang +++ /dev/null @@ -1,1188 +0,0 @@ -/* - * This module has been generated by smidump 0.4.8: - * - * smidump -f yang IF-MIB - * - * Do not edit. Edit the source file instead! - */ - -module IF-MIB { - - /*** NAMESPACE / PREFIX DEFINITION ***/ - - namespace "urn:ietf:params:xml:ns:yang:smiv2:IF-MIB"; - prefix "if-mib"; - - /*** LINKAGE (IMPORTS / INCLUDES) ***/ - - import IANAifType-MIB { prefix "ianaiftype-mib"; } - import SNMPv2-TC { prefix "smiv2"; } - import ietf-yang-types { prefix "yang"; } - - /*** META INFORMATION ***/ - - organization - "IETF Interfaces MIB Working Group"; - - contact - " Keith McCloghrie - Cisco Systems, Inc. - 170 West Tasman Drive - San Jose, CA 95134-1706 - US - - 408-526-5260 - kzm@cisco.com"; - - description - "The MIB module to describe generic objects for network - interface sub-layers. This MIB is an updated version of - MIB-II's ifTable, and incorporates the extensions defined in - RFC 1229."; - - revision "2000-06-14" { - description - "Clarifications agreed upon by the Interfaces MIB WG, and - published as RFC 2863."; - } - revision "1996-02-28" { - description - "Revisions made by the Interfaces MIB WG, and published in - RFC 2233."; - } - revision "1993-11-08" { - description - "Initial revision, published as part of RFC 1573."; - } - - /*** TYPE DEFINITIONS ***/ - - typedef OwnerString { - type string { - length "0..255"; - pattern "\p{IsBasicLatin}{0,255}"; - } - status deprecated; - description - "This data type is used to model an administratively - assigned name of the owner of a resource. This information - is taken from the NVT ASCII character set. It is suggested - that this name contain one or more of the following: ASCII - form of the manager station's transport address, management - station name (e.g., domain name), network management - personnel's name, location, or phone number. In some cases - the agent itself will be the owner of an entry. In these - cases, this string shall be set to a string starting with - 'agent'."; - } - - typedef InterfaceIndex { - type int32 { - range "1..2147483647"; - } - description - "A unique value, greater than zero, for each interface or - interface sub-layer in the managed system. It is - recommended that values are assigned contiguously starting - from 1. The value for each interface sub-layer must remain - constant at least from one re-initialization of the entity's - network management system to the next re-initialization."; - } - - typedef InterfaceIndexOrZero { - type int32 { - range "0..2147483647"; - } - description - "This textual convention is an extension of the - InterfaceIndex convention. The latter defines a greater - than zero value used to identify an interface or interface - sub-layer in the managed system. This extension permits the - additional value of zero. the value zero is object-specific - and must therefore be defined as part of the description of - any object which uses this syntax. Examples of the usage of - zero might include situations where interface was unknown, - or when none or all interfaces need to be referenced."; - } - - container interfaces { - - leaf ifNumber { - type int32; - config false; - description - "The number of network interfaces (regardless of their - current state) present on this system."; - } - - - /* XXX table comments here XXX */ - - list ifEntry { - key "ifIndex"; - description - "An entry containing management information applicable to a - particular interface."; - - - leaf ifIndex { - type if-mib:InterfaceIndex; - description - "A unique value, greater than zero, for each interface. It - is recommended that values are assigned contiguously - starting from 1. The value for each interface sub-layer - must remain constant at least from one re-initialization of - the entity's network management system to the next re- - initialization."; - } - - leaf ifDescr { - type smiv2:DisplayString { - length "0..255"; - } - config false; - description - "A textual string containing information about the - interface. This string should include the name of the - manufacturer, the product name and the version of the - interface hardware/software."; - } - - leaf ifType { - type ianaiftype-mib:IANAifType; - config false; - description - "The type of interface. Additional values for ifType are - assigned by the Internet Assigned Numbers Authority (IANA), - through updating the syntax of the IANAifType textual - convention."; - } - - leaf ifMtu { - type int32; - config false; - description - "The size of the largest packet which can be sent/received - on the interface, specified in octets. For interfaces that - are used for transmitting network datagrams, this is the - size of the largest network datagram that can be sent on the - interface."; - } - - leaf ifSpeed { - type yang:gauge32; - config false; - description - "An estimate of the interface's current bandwidth in bits - per second. For interfaces which do not vary in bandwidth - or for those where no accurate estimation can be made, this - object should contain the nominal bandwidth. If the - bandwidth of the interface is greater than the maximum value - reportable by this object then this object should report its - maximum value (4,294,967,295) and ifHighSpeed must be used - to report the interace's speed. For a sub-layer which has - no concept of bandwidth, this object should be zero."; - } - - leaf ifPhysAddress { - type yang:phys-address; - config false; - description - "The interface's address at its protocol sub-layer. For - example, for an 802.x interface, this object normally - contains a MAC address. The interface's media-specific MIB - must define the bit and byte ordering and the format of the - value of this object. For interfaces which do not have such - an address (e.g., a serial line), this object should contain - an octet string of zero length."; - } - - leaf ifAdminStatus { - type enumeration { - enum up { value 1; } - enum down { value 2; } - enum testing { value 3; } - } - config true; - description - "The desired state of the interface. The testing(3) state - indicates that no operational packets can be passed. When a - managed system initializes, all interfaces start with - ifAdminStatus in the down(2) state. As a result of either - explicit management action or per configuration information - retained by the managed system, ifAdminStatus is then - changed to either the up(1) or testing(3) states (or remains - in the down(2) state)."; - } - - leaf ifOperStatus { - type enumeration { - enum up { value 1; } - enum down { value 2; } - enum testing { value 3; } - enum unknown { value 4; } - enum dormant { value 5; } - enum notPresent { value 6; } - enum lowerLayerDown { value 7; } - } - config false; - description - "The current operational state of the interface. The - testing(3) state indicates that no operational packets can - be passed. If ifAdminStatus is down(2) then ifOperStatus - should be down(2). If ifAdminStatus is changed to up(1) - then ifOperStatus should change to up(1) if the interface is - ready to transmit and receive network traffic; it should - change to dormant(5) if the interface is waiting for - external actions (such as a serial line waiting for an - incoming connection); it should remain in the down(2) state - if and only if there is a fault that prevents it from going - to the up(1) state; it should remain in the notPresent(6) - state if the interface has missing (typically, hardware) - components."; - } - - leaf ifLastChange { - type yang:timeticks; - config false; - description - "The value of sysUpTime at the time the interface entered - its current operational state. If the current state was - entered prior to the last re-initialization of the local - network management subsystem, then this object contains a - zero value."; - } - - leaf ifInOctets { - type yang:counter32; - config false; - description - "The total number of octets received on the interface, - - - including framing characters. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifInUcastPkts { - type yang:counter32; - config false; - description - "The number of packets, delivered by this sub-layer to a - higher (sub-)layer, which were not addressed to a multicast - or broadcast address at this sub-layer. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifInNUcastPkts { - type yang:counter32; - config false; - status deprecated; - description - "The number of packets, delivered by this sub-layer to a - higher (sub-)layer, which were addressed to a multicast or - broadcast address at this sub-layer. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime. - - This object is deprecated in favour of ifInMulticastPkts and - ifInBroadcastPkts."; - } - - leaf ifInDiscards { - type yang:counter32; - config false; - description - "The number of inbound packets which were chosen to be - discarded even though no errors had been detected to prevent - - - their being deliverable to a higher-layer protocol. One - possible reason for discarding such a packet could be to - free up buffer space. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifInErrors { - type yang:counter32; - config false; - description - "For packet-oriented interfaces, the number of inbound - packets that contained errors preventing them from being - deliverable to a higher-layer protocol. For character- - oriented or fixed-length interfaces, the number of inbound - transmission units that contained errors preventing them - from being deliverable to a higher-layer protocol. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifInUnknownProtos { - type yang:counter32; - config false; - description - "For packet-oriented interfaces, the number of packets - received via the interface which were discarded because of - an unknown or unsupported protocol. For character-oriented - or fixed-length interfaces that support protocol - multiplexing the number of transmission units received via - the interface which were discarded because of an unknown or - unsupported protocol. For any interface that does not - support protocol multiplexing, this counter will always be - 0. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifOutOctets { - type yang:counter32; - config false; - description - "The total number of octets transmitted out of the - interface, including framing characters. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifOutUcastPkts { - type yang:counter32; - config false; - description - "The total number of packets that higher-level protocols - requested be transmitted, and which were not addressed to a - multicast or broadcast address at this sub-layer, including - those that were discarded or not sent. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifOutNUcastPkts { - type yang:counter32; - config false; - status deprecated; - description - "The total number of packets that higher-level protocols - requested be transmitted, and which were addressed to a - multicast or broadcast address at this sub-layer, including - those that were discarded or not sent. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime. - - This object is deprecated in favour of ifOutMulticastPkts - and ifOutBroadcastPkts."; - } - - leaf ifOutDiscards { - type yang:counter32; - config false; - description - "The number of outbound packets which were chosen to be - discarded even though no errors had been detected to prevent - their being transmitted. One possible reason for discarding - such a packet could be to free up buffer space. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifOutErrors { - type yang:counter32; - config false; - description - "For packet-oriented interfaces, the number of outbound - packets that could not be transmitted because of errors. - For character-oriented or fixed-length interfaces, the - number of outbound transmission units that could not be - transmitted because of errors. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifOutQLen { - type yang:gauge32; - config false; - status deprecated; - description - "The length of the output packet queue (in packets)."; - } - - leaf ifSpecific { - type yang:object-identifier; - config false; - status deprecated; - description - "A reference to MIB definitions specific to the particular - media being used to realize the interface. It is - - - recommended that this value point to an instance of a MIB - object in the media-specific MIB, i.e., that this object - have the semantics associated with the InstancePointer - textual convention defined in RFC 2579. In fact, it is - recommended that the media-specific MIB specify what value - ifSpecific should/can take for values of ifType. If no MIB - definitions specific to the particular media are available, - the value should be set to the OBJECT IDENTIFIER { 0 0 }."; - } - } - } - - container ifMIBObjects { - - - /* XXX table comments here XXX */ - - list ifStackEntry { - key "ifStackHigherLayer ifStackLowerLayer"; - description - "Information on a particular relationship between two sub- - layers, specifying that one sub-layer runs on 'top' of the - other sub-layer. Each sub-layer corresponds to a conceptual - row in the ifTable."; - - - leaf ifStackHigherLayer { - type if-mib:InterfaceIndexOrZero; - description - "The value of ifIndex corresponding to the higher sub-layer - of the relationship, i.e., the sub-layer which runs on 'top' - of the sub-layer identified by the corresponding instance of - ifStackLowerLayer. If there is no higher sub-layer (below - the internetwork layer), then this object has the value 0."; - } - - leaf ifStackLowerLayer { - type if-mib:InterfaceIndexOrZero; - description - "The value of ifIndex corresponding to the lower sub-layer - of the relationship, i.e., the sub-layer which runs 'below' - the sub-layer identified by the corresponding instance of - ifStackHigherLayer. If there is no lower sub-layer, then - this object has the value 0."; - } - - leaf ifStackStatus { - type smiv2:RowStatus; - config true; - description - "The status of the relationship between two sub-layers. - - Changing the value of this object from 'active' to - 'notInService' or 'destroy' will likely have consequences up - and down the interface stack. Thus, write access to this - object is likely to be inappropriate for some types of - interfaces, and many implementations will choose not to - support write-access for any type of interface."; - } - } - - - - /* XXX table comments here XXX */ - - list ifRcvAddressEntry { - key "ifIndex ifRcvAddressAddress"; - description - "A list of objects identifying an address for which the - system will accept packets/frames on the particular - interface identified by the index value ifIndex."; - - leaf ifIndex { - type leafref { - path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; - } - description - "Automagically generated leafref leaf."; - } - - leaf ifRcvAddressAddress { - type yang:phys-address; - description - "An address for which the system will accept packets/frames - on this entry's interface."; - } - - leaf ifRcvAddressStatus { - type smiv2:RowStatus; - config true; - description - "This object is used to create and delete rows in the - ifRcvAddressTable."; - } - - leaf ifRcvAddressType { - type enumeration { - enum other { value 1; } - enum volatile { value 2; } - enum nonVolatile { value 3; } - } - config true; - description - "This object has the value nonVolatile(3) for those entries - in the table which are valid and will not be deleted by the - next restart of the managed system. Entries having the - value volatile(2) are valid and exist, but have not been - saved, so that will not exist after the next restart of the - managed system. Entries having the value other(1) are valid - and exist but are not classified as to whether they will - continue to exist after the next restart."; - } - } - - leaf ifTableLastChange { - type yang:timeticks; - config false; - description - "The value of sysUpTime at the time of the last creation or - deletion of an entry in the ifTable. If the number of - entries has been unchanged since the last re-initialization - of the local network management subsystem, then this object - contains a zero value."; - } - - leaf ifStackLastChange { - type yang:timeticks; - config false; - description - "The value of sysUpTime at the time of the last change of - the (whole) interface stack. A change of the interface - stack is defined to be any creation, deletion, or change in - value of any instance of ifStackStatus. If the interface - stack has been unchanged since the last re-initialization of - the local network management subsystem, then this object - contains a zero value."; - } - } - - - /* XXX table comments here XXX */ - - augment "/if-mib:interfaces/if-mib:ifEntry" { - description - "An entry containing additional management information - applicable to a particular interface."; - - leaf ifName { - type smiv2:DisplayString; - config false; - description - "The textual name of the interface. The value of this - object should be the name of the interface as assigned by - the local device and should be suitable for use in commands - entered at the device's `console'. This might be a text - name, such as `le0' or a simple port number, such as `1', - depending on the interface naming syntax of the device. If - several entries in the ifTable together represent a single - interface as named by the device, then each will have the - same value of ifName. Note that for an agent which responds - to SNMP queries concerning an interface on some other - (proxied) device, then the value of ifName for such an - interface is the proxied device's local name for it. - - If there is no local name, or this object is otherwise not - applicable, then this object contains a zero-length string."; - } - - leaf ifInMulticastPkts { - type yang:counter32; - config false; - description - "The number of packets, delivered by this sub-layer to a - higher (sub-)layer, which were addressed to a multicast - address at this sub-layer. For a MAC layer protocol, this - includes both Group and Functional addresses. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - - - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifInBroadcastPkts { - type yang:counter32; - config false; - description - "The number of packets, delivered by this sub-layer to a - higher (sub-)layer, which were addressed to a broadcast - address at this sub-layer. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifOutMulticastPkts { - type yang:counter32; - config false; - description - "The total number of packets that higher-level protocols - requested be transmitted, and which were addressed to a - multicast address at this sub-layer, including those that - were discarded or not sent. For a MAC layer protocol, this - includes both Group and Functional addresses. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifOutBroadcastPkts { - type yang:counter32; - config false; - description - "The total number of packets that higher-level protocols - requested be transmitted, and which were addressed to a - broadcast address at this sub-layer, including those that - were discarded or not sent. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - - - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCInOctets { - type yang:counter64; - config false; - description - "The total number of octets received on the interface, - including framing characters. This object is a 64-bit - version of ifInOctets. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCInUcastPkts { - type yang:counter64; - config false; - description - "The number of packets, delivered by this sub-layer to a - higher (sub-)layer, which were not addressed to a multicast - or broadcast address at this sub-layer. This object is a - 64-bit version of ifInUcastPkts. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCInMulticastPkts { - type yang:counter64; - config false; - description - "The number of packets, delivered by this sub-layer to a - higher (sub-)layer, which were addressed to a multicast - address at this sub-layer. For a MAC layer protocol, this - includes both Group and Functional addresses. This object - is a 64-bit version of ifInMulticastPkts. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCInBroadcastPkts { - type yang:counter64; - config false; - description - "The number of packets, delivered by this sub-layer to a - higher (sub-)layer, which were addressed to a broadcast - address at this sub-layer. This object is a 64-bit version - of ifInBroadcastPkts. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCOutOctets { - type yang:counter64; - config false; - description - "The total number of octets transmitted out of the - interface, including framing characters. This object is a - 64-bit version of ifOutOctets. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCOutUcastPkts { - type yang:counter64; - config false; - description - "The total number of packets that higher-level protocols - requested be transmitted, and which were not addressed to a - multicast or broadcast address at this sub-layer, including - those that were discarded or not sent. This object is a - 64-bit version of ifOutUcastPkts. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCOutMulticastPkts { - type yang:counter64; - config false; - description - "The total number of packets that higher-level protocols - requested be transmitted, and which were addressed to a - multicast address at this sub-layer, including those that - were discarded or not sent. For a MAC layer protocol, this - includes both Group and Functional addresses. This object - is a 64-bit version of ifOutMulticastPkts. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifHCOutBroadcastPkts { - type yang:counter64; - config false; - description - "The total number of packets that higher-level protocols - requested be transmitted, and which were addressed to a - broadcast address at this sub-layer, including those that - were discarded or not sent. This object is a 64-bit version - of ifOutBroadcastPkts. - - Discontinuities in the value of this counter can occur at - re-initialization of the management system, and at other - times as indicated by the value of - ifCounterDiscontinuityTime."; - } - - leaf ifLinkUpDownTrapEnable { - type enumeration { - enum enabled { value 1; } - enum disabled { value 2; } - } - config true; - description - "Indicates whether linkUp/linkDown traps should be generated - for this interface. - - By default, this object should have the value enabled(1) for - interfaces which do not operate on 'top' of any other - interface (as defined in the ifStackTable), and disabled(2) - otherwise."; - } - - leaf ifHighSpeed { - type yang:gauge32; - config false; - description - "An estimate of the interface's current bandwidth in units - of 1,000,000 bits per second. If this object reports a - value of `n' then the speed of the interface is somewhere in - the range of `n-500,000' to `n+499,999'. For interfaces - which do not vary in bandwidth or for those where no - accurate estimation can be made, this object should contain - the nominal bandwidth. For a sub-layer which has no concept - of bandwidth, this object should be zero."; - } - - leaf ifPromiscuousMode { - type smiv2:TruthValue; - config true; - description - "This object has a value of false(2) if this interface only - accepts packets/frames that are addressed to this station. - This object has a value of true(1) when the station accepts - all packets/frames transmitted on the media. The value - true(1) is only legal on certain types of media. If legal, - setting this object to a value of true(1) may require the - interface to be reset before becoming effective. - - The value of ifPromiscuousMode does not affect the reception - of broadcast and multicast packets/frames by the interface."; - } - - leaf ifConnectorPresent { - type smiv2:TruthValue; - config false; - description - "This object has the value 'true(1)' if the interface - sublayer has a physical connector and the value 'false(2)' - otherwise."; - } - - leaf ifAlias { - type smiv2:DisplayString { - length "0..64"; - } - config true; - description - "This object is an 'alias' name for the interface as - specified by a network manager, and provides a non-volatile - 'handle' for the interface. - - On the first instantiation of an interface, the value of - ifAlias associated with that interface is the zero-length - string. As and when a value is written into an instance of - ifAlias through a network management set operation, then the - agent must retain the supplied value in the ifAlias instance - associated with the same interface for as long as that - interface remains instantiated, including across all re- - initializations/reboots of the network management system, - including those which result in a change of the interface's - ifIndex value. - - An example of the value which a network manager might store - in this object for a WAN interface is the (Telco's) circuit - number/identifier of the interface. - - Some agents may support write-access only for interfaces - having particular values of ifType. An agent which supports - write access to this object is required to keep the value in - non-volatile storage, but it may limit the length of new - values depending on how much storage is already occupied by - the current values for other interfaces."; - } - - leaf ifCounterDiscontinuityTime { - type yang:timestamp; - config false; - description - "The value of sysUpTime on the most recent occasion at which - any one or more of this interface's counters suffered a - discontinuity. The relevant counters are the specific - instances associated with this interface of any Counter32 or - - - Counter64 object contained in the ifTable or ifXTable. If - no such discontinuities have occurred since the last re- - initialization of the local management subsystem, then this - object contains a zero value."; - } - } - - - /* XXX table comments here XXX */ - - augment "/if-mib:interfaces/if-mib:ifEntry" { - status deprecated; - description - "An entry containing objects for invoking tests on an - interface."; - - leaf ifTestId { - type smiv2:TestAndIncr; - config true; - status deprecated; - description - "This object identifies the current invocation of the - interface's test."; - } - - leaf ifTestStatus { - type enumeration { - enum notInUse { value 1; } - enum inUse { value 2; } - } - config true; - status deprecated; - description - "This object indicates whether or not some manager currently - has the necessary 'ownership' required to invoke a test on - this interface. A write to this object is only successful - when it changes its value from 'notInUse(1)' to 'inUse(2)'. - After completion of a test, the agent resets the value back - to 'notInUse(1)'."; - } - - leaf ifTestType { - type smiv2:AutonomousType; - config true; - status deprecated; - description - "A control variable used to start and stop operator- - initiated interface tests. Most OBJECT IDENTIFIER values - assigned to tests are defined elsewhere, in association with - specific types of interface. However, this document assigns - a value for a full-duplex loopback test, and defines the - special meanings of the subject identifier: - - noTest OBJECT IDENTIFIER ::= { 0 0 } - - When the value noTest is written to this object, no action - is taken unless a test is in progress, in which case the - test is aborted. Writing any other value to this object is - - - only valid when no test is currently in progress, in which - case the indicated test is initiated. - - When read, this object always returns the most recent value - that ifTestType was set to. If it has not been set since - the last initialization of the network management subsystem - on the agent, a value of noTest is returned."; - } - - leaf ifTestResult { - type enumeration { - enum none { value 1; } - enum success { value 2; } - enum inProgress { value 3; } - enum notSupported { value 4; } - enum unAbleToRun { value 5; } - enum aborted { value 6; } - enum failed { value 7; } - } - config false; - status deprecated; - description - "This object contains the result of the most recently - requested test, or the value none(1) if no tests have been - requested since the last reset. Note that this facility - provides no provision for saving the results of one test - when starting another, as could be required if used by - multiple managers concurrently."; - } - - leaf ifTestCode { - type yang:object-identifier; - config false; - status deprecated; - description - "This object contains a code which contains more specific - information on the test result, for example an error-code - after a failed test. Error codes and other values this - object may take are specific to the type of interface and/or - test. The value may have the semantics of either the - AutonomousType or InstancePointer textual conventions as - defined in RFC 2579. The identifier: - - testCodeUnknown OBJECT IDENTIFIER ::= { 0 0 } - - is defined for use if no additional result code is - available."; - } - - leaf ifTestOwner { - type if-mib:OwnerString; - config true; - status deprecated; - description - "The entity which currently has the 'ownership' required to - invoke a test on this interface."; - } - } - - notification linkDown { - description - "A linkDown trap signifies that the SNMP entity, acting in - an agent role, has detected that the ifOperStatus object for - one of its communication links is about to enter the down - state from some other state (but not from the notPresent - state). This other state is indicated by the included value - of ifOperStatus."; - - container linkDown-ifIndex { - leaf ifIndex { - type leafref { - path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; - } - description - "Automagically generated leafref leaf."; - } - } - - container linkDown-ifAdminStatus { - leaf ifIndex { - type leafref { - path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; - } - description - "Automagically generated leafref leaf."; - } - leaf ifAdminStatus { - type enumeration { - enum up { value 1; } - enum down { value 2; } - enum testing { value 3; } - } - description - "The desired state of the interface. The testing(3) state - indicates that no operational packets can be passed. When a - managed system initializes, all interfaces start with - ifAdminStatus in the down(2) state. As a result of either - explicit management action or per configuration information - retained by the managed system, ifAdminStatus is then - changed to either the up(1) or testing(3) states (or remains - in the down(2) state)."; - } - } - - container linkDown-ifOperStatus { - leaf ifIndex { - type leafref { - path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; - } - description - "Automagically generated leafref leaf."; - } - leaf ifOperStatus { - type enumeration { - enum up { value 1; } - enum down { value 2; } - enum testing { value 3; } - enum unknown { value 4; } - enum dormant { value 5; } - enum notPresent { value 6; } - enum lowerLayerDown { value 7; } - } - description - "The current operational state of the interface. The - testing(3) state indicates that no operational packets can - be passed. If ifAdminStatus is down(2) then ifOperStatus - should be down(2). If ifAdminStatus is changed to up(1) - then ifOperStatus should change to up(1) if the interface is - ready to transmit and receive network traffic; it should - change to dormant(5) if the interface is waiting for - external actions (such as a serial line waiting for an - incoming connection); it should remain in the down(2) state - if and only if there is a fault that prevents it from going - to the up(1) state; it should remain in the notPresent(6) - state if the interface has missing (typically, hardware) - components."; - } - } - - } - - notification linkUp { - description - "A linkUp trap signifies that the SNMP entity, acting in an - agent role, has detected that the ifOperStatus object for - one of its communication links left the down state and - transitioned into some other state (but not into the - notPresent state). This other state is indicated by the - included value of ifOperStatus."; - - container linkUp-ifIndex { - leaf ifIndex { - type leafref { - path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; - } - description - "Automagically generated leafref leaf."; - } - } - - container linkUp-ifAdminStatus { - leaf ifIndex { - type leafref { - path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; - } - description - "Automagically generated leafref leaf."; - } - leaf ifAdminStatus { - type enumeration { - enum up { value 1; } - enum down { value 2; } - enum testing { value 3; } - } - description - "The desired state of the interface. The testing(3) state - indicates that no operational packets can be passed. When a - managed system initializes, all interfaces start with - ifAdminStatus in the down(2) state. As a result of either - explicit management action or per configuration information - retained by the managed system, ifAdminStatus is then - changed to either the up(1) or testing(3) states (or remains - in the down(2) state)."; - } - } - - container linkUp-ifOperStatus { - leaf ifIndex { - type leafref { - path "/if-mib:interfaces/if-mib:ifEntry/if-mib:ifIndex"; - } - description - "Automagically generated leafref leaf."; - } - leaf ifOperStatus { - type enumeration { - enum up { value 1; } - enum down { value 2; } - enum testing { value 3; } - enum unknown { value 4; } - enum dormant { value 5; } - enum notPresent { value 6; } - enum lowerLayerDown { value 7; } - } - description - "The current operational state of the interface. The - testing(3) state indicates that no operational packets can - be passed. If ifAdminStatus is down(2) then ifOperStatus - should be down(2). If ifAdminStatus is changed to up(1) - then ifOperStatus should change to up(1) if the interface is - ready to transmit and receive network traffic; it should - change to dormant(5) if the interface is waiting for - external actions (such as a serial line waiting for an - incoming connection); it should remain in the down(2) state - if and only if there is a fault that prevents it from going - to the up(1) state; it should remain in the notPresent(6) - state if the interface has missing (typically, hardware) - components."; - } - } - - } -} /* end of module IF-MIB */ \ No newline at end of file From 6c9fc4ea2e24e5e2d079ecbcae890dde9c6fde39 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 25 Oct 2016 15:26:00 +0200 Subject: [PATCH 051/631] Add IETF softwire schema and test it --- src/lib/yang/ietf-softwire.yang | 778 ++++++++++++++++++++++++++++++++ src/lib/yang/schema2.lua | 5 +- 2 files changed, 782 insertions(+), 1 deletion(-) create mode 100644 src/lib/yang/ietf-softwire.yang diff --git a/src/lib/yang/ietf-softwire.yang b/src/lib/yang/ietf-softwire.yang new file mode 100644 index 0000000000..ae4490df14 --- /dev/null +++ b/src/lib/yang/ietf-softwire.yang @@ -0,0 +1,778 @@ +module ietf-softwire { + namespace "urn:ietf:params:xml:ns:yang:ietf-softwire"; + prefix "softwire"; + + import ietf-inet-types {prefix inet; } + import ietf-yang-types {prefix yang; } + + organization "Softwire Working Group"; + + contact + " + Qi Sun + Hao Wang + Yong Cui + Ian + Sladjana Zoric + Mohamed Boucadair + Rajiv + "; + + description + "This document defines a YANG data model for the configuration and + management of A+P Softwire Border Routers (BRs) and Customer + Premises Equipment (CEs). It covers Lightweight 4over6, + MAP-E and MAP-T mechanisms. + + Copyright (c) 2016 IETF Trust and the persons identified + as authors of the code. All rights reserved. + This version of this YANG module is part of RFC XXX; see the RFC + itself for full legal notices."; + + revision 2016-06-04 { + description + "Version-05: Combined MAP-E/MAP-T into a single tree. Added binding + table/alogorthm versioning"; + reference "-05"; + } + + revision 2015-09-30 { + description + "Version-04: Fix YANG syntax; Add flags to map-rule; Remove + the map-rule-type element. "; + reference "-04"; + } + + revision 2015-04-07 { + description + "Version-03: Integrate lw4over6; Updata state nodes; Correct + grammar errors; Reuse groupings; Update descriptions. + Simplify the model."; + reference "-03"; + } + + revision 2015-02-10 { + description + "Version-02: Add notifications."; + reference "-02"; + } + + + revision 2015-02-06 { + description + "Version-01: Correct grammar errors; Reuse groupings; Update + descriptions."; + reference "-01"; + } + + revision 2015-02-02 { + description + "Initial revision."; + reference "-00"; + } + +/* + * Features + */ + + feature binding { + description + "Lightweight 4over6 (binding) is an IPv4-over-IPv6 tunnelling + transition mechanism. Lightweight 4over6 is a solution designed + specifically for complete independence between IPv6 subnet + prefix (and /128 IPv6 address) and IPv4 address with or + without IPv4 address sharing. + + This is accomplished by maintaining state for + each softwire (per-subscriber state) in the central lwAFTR and + a hub-and-spoke forwarding architecture. In order to delegate + the NAPT function and achieve IPv4 address sharing, + port-restricted IPv4 addresses needs to be allocated to CEs. + + Besides lw4o6, this feature also covers MAP in 1:1 mode + (offset=0, PSID explicit)"; + + reference + "RFC7596"; + } + + feature br { + if-feature binding; + description + "The AFTR for Lightweight 4over6, so-called lwAFTR (BR). This + feature indicates that a instance functions as a lwAFTR (BR). + A lwAFTR (BR) is an IPv4-in-IPv6 tunnel concentrator that + maintains per-subscriber IPv4-IPv6 address binding."; + } + + feature ce { + if-feature binding; + description + "The B4 for Lightweight 4over6, so-called lwB4 (CE). This + feature indicates that a instance functions as a lwB4 (CE). A + lwB4 (ce) is an IPv4-in-IPv6 tunnel initiator. It is + dual-stack capable node, either a directly connected end-host + or a CE. It sources IPv4 connections using the configured + port-set and the public IPv4 address."; + } + + feature algorithm { + description + "MAP-E is an IPv6 transition mechanism for transporting IPv4 + packets across an IPv6 network using IP encapsulation. MAP-E + allows for a reduction of the amount of centralized state using + rules to express IPv4/IPv6 address mappings. This introduces an + algorithmic relationship between the IPv6 subnet + and IPv4 address. + The Mapping of Address and Port - Translation (MAP-T) + architecture is a double stateless NAT64 based solution. It uses + the stateless algorithmic address & transport layer port mapping + scheme defined in MAP-E. The MAP-T solution differs from MAP-E in + the use of IPv4-IPv6 translation, rather than encapsulation, as + the form of IPv6 domain transport. + This feature indicates the instance functions as a MAP-E or + MAP-T instance."; + reference + "RFC7597 & RFC7599"; + } + +/* + * Grouping + */ + + grouping port-set { + description + "Use the PSID algorithm to represent a range of transport layer + ports."; + leaf psid-offset { + type uint8 { + range 0..16; + } + mandatory true; + description + "The number of offset bits. In Lightweight 4over6, the default + value is 0 for assigning one contiguous port range. In MAP-E/T, + the default value is 6, which excludes system ports by default + and assigns distributed port ranges. If the this parameter is + larger than 0, the value of offset MUST be greater than 0."; + } + leaf psid-len { + type uint8 { + range 0..15; + } + mandatory true; + description + "The length of PSID, representing the sharing ratio for an + IPv4 address."; + } + leaf psid { + type uint16; + mandatory true; + description + "Port Set Identifier (PSID) value, which identifies a set + of ports algorithmically."; + } + } + + grouping binding-entry { + description + "The lwAFTR maintains an address binding table that contains + the binding between the lwB4's IPv6 address, the allocated IPv4 + address and restricted port-set."; + leaf binding-ipv6info { + type union { + type inet:ipv6-address; + type inet:ipv6-prefix; + } + mandatory true; + description + "The IPv6 information for a binding entry. + If it's an IPv6 prefix, it indicates that + the IPv6 source address of the lwB4 is constructed + according to the description in RFC7596; + if it's an IPv6 address, it means the lwB4 uses + any /128 address from the assigned IPv6 prefix. + "; + } + leaf binding-ipv4-addr { + type inet:ipv4-address; + mandatory true; + description + "The IPv4 address assigned to the lwB4, which is + used as the IPv4 external address + for lwB4 local NAPT44."; + } + container port-set { + description + "For Lightweight 4over6, the default value + of offset should be 0, to configure one contiguous + port range."; + uses port-set { + refine psid-offset { + default "0"; + } + } + } + leaf br-ipv6-addr { + type inet:ipv6-address; + mandatory true; + description + "The IPv6 address for lwaftr."; + } + leaf lifetime { + type uint32; + units seconds; + description "The lifetime for the binding entry"; + } + } + +/* + grouping nat-table { + + description + "Grouping 'nat-table' is not extended. The current mechanism + is focusing on the provisioning of external IP address and + port set; other NAT-specific considerations are out of scope."; + } +*/ + + grouping traffic-stat { + description "Traffic statistics"; + leaf sentPacket { + type yang:zero-based-counter64; + description "Number of packets sent."; + } + leaf sentByte { + type yang:zero-based-counter64; + description "Traffic sent, in bytes"; + } + leaf rcvdPacket { + type yang:zero-based-counter64; + description "Number of packets received."; + } + leaf rcvdByte { + type yang:zero-based-counter64; + description "Traffic received, in bytes"; + } + leaf droppedPacket { + type yang:zero-based-counter64; + description "Number of packets dropped."; + } + leaf droppedByte { + type yang:zero-based-counter64; + description "Traffic dropped, in bytes"; + } + } + + +/* + * Configuration Data Nodes + */ + + + container softwire-config { + description + "The configuration data for Softwire instances. And the shared + data describes the softwire data model which is common to all of + the different softwire mechanisms, such as description."; + leaf description { + type string; + description + "A textual description of Softwire."; + } + container binding { + if-feature binding; + description + "lw4over6 (binding) configuration."; + container br { + if-feature br; + description + "Indicate this instance supports the lwAFTR (BR) function. + The instances advertise the BR feature through the + capability exchange mechanism when a NETCONF session is + established."; + leaf enable { + type boolean; + description + "Enable/disable the lwAFTR (BR) function."; + } + container br-instances { + description + "A set of BRs to be configured."; + list br-instance { + key "id"; + description + "A set of lwAFTRs to be configured."; + leaf id { + type uint32; + mandatory true; + description "An instance identifier."; + } + container binding-table-version { + description "binding table's version"; + leaf binding-table-version{ + type uint64; + description "Incremental version number + to the binding table"; + } + leaf binding-table-date { + type yang:date-and-time; + description "Timestamp to the binding + table"; + } + } + leaf name { + type string; + description "The name for the lwaftr."; + } + leaf softwire-num-threshold { + type uint32; + mandatory true; + description + "The maximum number of tunnels that can be created on + the lwAFTR."; + } + leaf tunnel-payload-mtu { + type uint16; + mandatory true; + description + "The payload MTU for Lightweight 4over6 tunnel."; + } + leaf tunnel-path-mru { + type uint16; + mandatory true; + description + "The path MRU for Lightweight 4over6 tunnel."; + } + container binding-table { + description "binding table"; + list binding-entry { + key "binding-ipv6info"; + description "binding entry"; + uses binding-entry; + } + } + } + } + } + + container ce { + if-feature ce; + description + "Indicate this instance supports the lwB4 (CE) function. + The instances advertise the CE feature through the + capability exchange mechanism when a NETCONF session is + established."; + leaf enable { + type boolean; + description + "Enable/disable the lwB4 (CE) function."; + } + container ce-instances { + description + "A set of CEs to be configured."; + list ce-instance { + key "binding-ipv6info"; + description "instances for CE"; + uses binding-entry; + leaf name { + type string; + description "The CE's name."; + } + leaf tunnel-payload-mtu { + type uint16; + mandatory true; + description + "The payload MTU for Lightweight 4over6 tunnel."; + } + leaf tunnel-path-mru { + type uint16; + mandatory true; + description + "The path MRU for Lightweight 4over6 tunnel."; + } + leaf b4-ipv6-addr-format { + type boolean; + mandatory true; + description + "The format of lwB4 (CE) IPv6 address. If set to true, + it indicates that the IPv6 source address of the lwB4 + is constructed according to the description in + [RFC7596]; if set to false, the lwB4 (CE) + can use any /128 address from the assigned IPv6 + prefix."; + } + } + } + } + } + + container algorithm { + if-feature algorithm; + description + "Indicate the instances support the MAP-E and MAP-T function. + The instances advertise the map-e feature through the + capability exchange mechanism when a NETCONF session is + established."; + leaf enable { + type boolean; + description + "Enable/disable the MAP-E or MAP-T function."; + } + container algo-instances { + description + "A set of MAP-E or MAP-T instances to be configured, + applying to BRs and CEs. A MAP-E/T instance defines a MAP + domain comprising one or more MAP-CE and MAP-BR"; + list algo-instance { + key "id"; + description "instance for MAP-E/MAP-T"; + leaf id { + type uint32; + mandatory true; + description "Algorithm Instance ID"; + } + container algo-versioning { + description "algorithm's version"; + leaf algo-version { + type uint64; + description "Incremental version number to + the algorithm"; + } + leaf algo-date { + type yang:date-and-time; + description "Timestamp to the algorithm"; + } + } + leaf name { + type string; + description "The name for the instance."; + } + leaf data-plane { + type enumeration { + enum "encapsulation" { + description "encapsulation for MAP-E"; + } + enum "translation" { + description "translation for MAP-T"; + } + } + description + "Encapsulation is for MAP-E while translation is + for MAP-T"; + } + leaf ea-len { + type uint8; + mandatory true; + description + "Embedded Address (EA) bits are the IPv4 EA-bits + in the IPv6 address identify an IPv4 + prefix/address (or part thereof) or + a shared IPv4 address (or part thereof) + and a port-set identifier. + The length of the EA-bits is defined as + part of a MAP rule for a MAP domain."; + } + leaf rule-ipv6-prefix { + type inet:ipv6-prefix; + mandatory true; + description + "The Rule IPv6 prefix defined in the mapping rule."; + } + leaf rule-ipv4-prefix { + type inet:ipv4-prefix; + mandatory true; + description + "The Rule IPv4 prefix defined in the mapping rule."; + } + leaf forwarding { + type boolean; + mandatory true; + description + "This parameter specifies whether the rule may be used for + forwarding (FMR). If set, this rule is used as an FMR; + if not set, this rule is a BMR only and must not be used + for forwarding."; + } + leaf psid-offset { + type uint8 { + range 0..16; + } + mandatory true; + description + "The number of offset bits. In Lightweight 4over6, the default + value is 0 for assigning one contiguous port range. In MAP-E/T, + the default value is 6, which excludes system ports by default + and assigns distributed port ranges. If the this parameter is + larger than 0, the value of offset MUST be greater than 0."; + } + leaf psid-len { + type uint8 { + range 0..15; + } + mandatory true; + description + "The length of PSID, representing the sharing ratio for an + IPv4 address."; + } + leaf tunnel-payload-mtu { + type uint16; + description + "The payload MTU for MAP-E tunnel."; + } + leaf tunnel-path-mru { + type uint16; + description + "The path MRU for MAP-E tunnel."; + } + leaf br-ipv6-addr { + type inet:ipv6-address; + mandatory true; + description + "The IPv6 address of the MAP-E BR."; + } + leaf dmr-ipv6-prefix { + type inet:ipv6-prefix; + description + "The IPv6 prefix of the MAP-T BR. "; + } + } + } + } + } + +/* + * Operational state Data Nodes + */ + + container softwire-state { + config false; + description + "The operational state data for Softwire instances. "; + leaf description { + type string; + description + "A textual description of the softwire instances."; + } + container binding { + if-feature binding; + description + "lw4over6 (binding) state."; + container br { + if-feature br; + config false; + description + "Indicate this instance supports the lwAFTR (BR) function. + The instances advertise the lwaftr (BR) feature through the + capability exchange mechanism when a NETCONF session is + established."; + container br-instances { + description + "A set of BRs."; + list br-instance { + key "id"; + description "instances for BR"; + leaf id { + type uint32; + mandatory true; + description "id"; + } + leaf name { + type string; + description "The name for this lwaftr."; + } + uses traffic-stat; + leaf active-softwire-num { + type uint32; + description + "The number of currently active tunnels on the + lw4over6 (binding) instance."; + } + container binding-table { + description "id"; + list binding-entry { + key "binding-ipv6info"; + description "An identifier of the binding entry."; + leaf binding-ipv6info { + type union { + type inet:ipv6-address; + type inet:ipv6-prefix; + } + mandatory true; + description + "The IPv6 information used to identify + a binding entry. "; + } + leaf active { + type boolean; + description + "Status of a specific tunnel."; + } + } + } + } + } + } + + + container ce { + if-feature ce; + config false; + description + "Indicate this instance supports the lwB4 (CE) function. + The instances advertise the lwb4 (CE) feature through the + capability exchange mechanism when a NETCONF session is + established."; + container ce-instances { + description + "Status of the configured CEs."; + list ce-instance { + key "binding-ipv6info"; + description "a lwB4 (CE) instance."; + leaf binding-ipv6info { + type union { + type inet:ipv6-address; + type inet:ipv6-prefix; + } + mandatory true; + description + "The IPv6 information used to identify + a binding entry. "; + } + leaf name { + type string; + description "The CE's name."; + } + uses traffic-stat; + } + } + } + } + + container algorithm { + if-feature algorithm; + config false; + description + "Indicate the instances support the MAP-E and MAP-T function. + The instances advertise the map-e/map-t feature through the + capability exchange mechanism when a NETCONF session is + established."; + container algo-instances { + description + "Status of MAP-E instance(s)."; + list algo-instance { + key "id"; + description "Instances for algorithm"; + leaf id { + type uint32; + mandatory true; + description "id"; + } + leaf name { + type string; + description "The map-e instance name."; + + } + uses traffic-stat; + } + } + } + } + +/* + * Notifications + */ + notification softwire-br-event { + if-feature binding; + if-feature br; + description "Notification for BR."; + + leaf br-id { + type leafref { + path + "/softwire-state/binding/br/br-instances/" + + "br-instance/id"; + } + description "..."; + } + leaf-list invalid-entry { + type leafref { + path + "/softwire-config/binding/br/br-instances/" + + "br-instance[id=current()/../br-id]/" + + "binding-table/binding-entry/binding-ipv6info"; + } + description + "Notify the client that a specific binding entry has been + expired/invalid. The binding-ipv6info identifies an entry."; + } + leaf-list added-entry { + type inet:ipv6-address; + description + "Notify the client that a binding entry has been added. + The ipv6 address of that entry is the index. The client + get other information from the lwaftr about the entry + indexed by that ipv6 address. + "; + } + leaf-list modified-entry { + type leafref { + path + "/softwire-config/binding/br/br-instances/" + + "br-instance[id=current()/../br-id]/" + + "binding-table/binding-entry/binding-ipv6info"; + } + description "..."; + } + } + + notification softwire-ce-event { + if-feature binding; + if-feature ce; + description "CE notification"; + leaf ce-binding-ipv6-addr-change { + type inet:ipv6-address; + mandatory true; + description + "The source tunnel IPv6 address of the lwB4. + If 'b4-ipv6-addr-format' is false, or the lwb4's + binding-ipv6-address changes for any reason, + it SHOULD notify the NETCONF client."; + } + } + + notification softwire-algorithm-instance-event { + if-feature algorithm; + description "Notifications for MAP-E or MAP-T."; + leaf algo-id { + type leafref { + path + "/softwire-config/algorithm/algo-instances/algo-instance/id"; + } + mandatory true; + description "MAP-E or MAP-T event."; + } + leaf-list invalid-entry-id { + type leafref { + path + "/softwire-config/algorithm/algo-instances/algo-instance/id"; + } + description "Invalid entry event."; + } + leaf-list added-entry { + type leafref { + path + "/softwire-config/algorithm/algo-instances/algo-instance/id"; + } + description "Added entry."; + } + leaf-list modified-entry { + type leafref { + path + "/softwire-config/algorithm/algo-instances/algo-instance/id"; + } + description "Modified entry."; + } + } +} + diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua index edb1272fa3..016c077298 100644 --- a/src/lib/yang/schema2.lua +++ b/src/lib/yang/schema2.lua @@ -642,7 +642,9 @@ function resolve(schema, features) end end for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do - if not has_feature(env, feature) then return nil, env end + if not pcall(lookup, env, 'features', feature) then + return nil, env + end end for k,v in pairs(node.body or {}) do if v.kind == 'uses' then @@ -845,4 +847,5 @@ function selftest() parse_schema(require('lib.yang.ietf_inet_types_yang')) load_schema_by_name('ietf-yang-types') + load_schema_by_name('ietf-softwire') end From 33808e4d61018a7f00c47bb91cf976cc84ed4b54 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 25 Oct 2016 17:18:30 +0200 Subject: [PATCH 052/631] Resolve prefixed imported extensions, identities, etc --- src/lib/yang/schema2.lua | 41 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua index 016c077298..387ccbb66c 100644 --- a/src/lib/yang/schema2.lua +++ b/src/lib/yang/schema2.lua @@ -414,7 +414,6 @@ local function init_pattern(node) node.reference = maybe_child_property(node, 'reference', 'value') end local function init_range(node) - -- TODO: parse range string node.value = parse_range(node, require_argument(node)) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -628,18 +627,23 @@ function resolve(schema, features) local grouping, grouping_env = lookup(env, 'groupings', name) return visit(grouping, grouping_env) end + local function visit_top_level(node, env, prop) + assert(not env[prop]) + env[prop] = {} + for k,v in pairs(pop_prop(node, prop) or {}) do + env[prop][k] = visit(v, env) + end + end function visit(node, env) node, env = push_env(node, env) local when = pop_prop(node, 'when') if when then print('warning: assuming "when" condition to be true: '..when.value) end - for k,v in pairs(pop_prop(node, 'features') or {}) do - v = visit(v, env) - if v then - if not env.features then env.features = {} end - env.features[k] = v - end + if node.kind == 'module' or node.kind == 'submodule' then + visit_top_level(node, env, 'extensions') + visit_top_level(node, env, 'features') + visit_top_level(node, env, 'identities') end for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do if not pcall(lookup, env, 'features', feature) then @@ -683,24 +687,31 @@ function resolve(schema, features) for k,v in pairs(pop_prop(node, 'includes')) do local submodule = lookup(env, 'submodules', k) assert(submodule.belongs_to.id == node.id) - submodule = link(submodule, env) - merge_tables(module_env.extensions, submodule.env.extensions) - merge_tables(module_env.features, submodule.env.features) - merge_tables(module_env.identities, submodule.env.identities) - merge_tables(module_env.typedefs, submodule.env.typedefs) - merge_tables(module_env.groupings, submodule.env.groupings) + submodule, submodule_env = link(submodule, env) + merge_tables(module_env.extensions, submodule_env.extensions) + merge_tables(module_env.features, submodule_env.features) + merge_tables(module_env.identities, submodule_env.identities) + merge_tables(module_env.typedefs, submodule_env.typedefs) + merge_tables(module_env.groupings, submodule_env.groupings) merge_tables(module_body, submodule.body) end if node.prefix then assert(node.kind == 'module', node.kind) - module_env.prefixes[node.prefix] = module_env + module_env.prefixes[node.prefix] = node.namespace end for k,v in pairs(pop_prop(node, 'imports')) do assert(not module_env.prefixes[v.prefix], 'duplicate prefix') -- CHECKME: Discarding body from import, just importing env. -- Is this OK? local schema, env = load_schema_by_name(v.id, v.revision_date) - module_env.prefixes[v.prefix] = env + local prefix = v.prefix + module_env.prefixes[prefix] = schema.namespace + for _,prop in ipairs({'extensions', 'features', 'identities', + 'typedefs', 'groupings'}) do + for k,v in pairs(env[prop]) do + module_env[prop][prefix..':'..k] = v + end + end end node, env = visit(node, module_env) -- The typedefs, groupings, identities, and so on of this module From 3fbce439925f797a0ca37cdd2d6f8bda2991d4dc Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Tue, 25 Oct 2016 17:34:50 +0200 Subject: [PATCH 053/631] In the 'bench' command, use the same option name for the benchmark filename as in the other commands (#514) --- src/program/lwaftr/bench/README | 9 +++++---- src/program/lwaftr/bench/bench.lua | 11 +++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/program/lwaftr/bench/README b/src/program/lwaftr/bench/README index 5c45cb327d..fd007fb2e8 100644 --- a/src/program/lwaftr/bench/README +++ b/src/program/lwaftr/bench/README @@ -11,10 +11,11 @@ Usage: bench CONF IPV4-IN.PCAP IPV6-IN.PCAP rather than the default: Time (s),Decap. MPPS,Decap. Gbps,Encap. MPPS,Encap. Gbps - -f FILENAME, --filename FILENAME - The file or path name to which CSV data is written. - A simple filename or relative pathname will be based - on the current directory. Default is "bench.csv". + -b FILENAME, --bench-file FILENAME + The file or path name to which benchmark data is + written. A simple filename or relative pathname + will be based on the current directory. Default + is "bench.csv". -D DURATION, --duration DURATION Duration in seconds. diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index c2b3fee914..fb16d9c800 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -13,17 +13,16 @@ end function parse_args(args) local handlers = {} - local opts = {} - opts.filename = "bench.csv" + local opts = { bench_file = 'bench.csv' } function handlers.D(arg) opts.duration = assert(tonumber(arg), "duration must be a number") assert(opts.duration >= 0, "duration can't be negative") end - function handlers.f(arg) opts.filename = arg end + function handlers.b(arg) opts.bench_file = arg end function handlers.y() opts.hydra = true end function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "hyf:D:", { - help="h", hydra="y", filename="f", duration="D" }) + args = lib.dogetopt(args, handlers, "hyb:D:", { + help="h", hydra="y", ["bench-file"]="b", duration="D" }) if #args ~= 3 then show_usage(1) end return opts, unpack(args) end @@ -36,7 +35,7 @@ function run(args) setup.load_bench(c, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') app.configure(c) - local csv = csv_stats.CSVStatsTimer:new(opts.filename, opts.hydra) + local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) csv:add_app('sinkv4', { 'input' }, { input=opts.hydra and 'decap' or 'Decap.' }) csv:add_app('sinkv6', { 'input' }, { input=opts.hydra and 'encap' or 'Encap.' }) csv:activate() From 2744a4069e234b068f0931da7fe540c54ee80a80 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 25 Oct 2016 20:03:12 +0200 Subject: [PATCH 054/631] Lazily resolve typedefs and groupings --- src/lib/yang/schema2.lua | 130 ++++++++++++++++++++++++++++++++------- 1 file changed, 108 insertions(+), 22 deletions(-) diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua index 387ccbb66c..32fef89936 100644 --- a/src/lib/yang/schema2.lua +++ b/src/lib/yang/schema2.lua @@ -591,6 +591,41 @@ local function strip(exp) return ret end +local function integer_type(min, max) return function(node) return node end end +local function binary_type(node) return node end +local function bits_type(node) return node end +local function boolean_type(node) return node end +local function decimal64_type(node) return node end +local function empty_type(node) return node end +local function enumeration_type(node) return node end +local function identityref_type(node) return node end +local function instance_identifier_type(node) return node end +local function leafref_type(node) return node end +local function string_type(node) return node end +local function union_type(node) return node end + +local primitive_type_constructors = { + int8 = integer_type(-0xf0, 0x7f), + int16 = integer_type(-0xf000, 0x7fff), + int32 = integer_type(-0xf000000, 0x7fffffff), + int64 = integer_type(-0xf00000000000000LL, 0x7fffffffffffffffLL), + uint8 = integer_type(0, 0xff), + uint16 = integer_type(0, 0xffff), + uint32 = integer_type(0, 0xffffffff), + uint64 = integer_type(0, 0xffffffffffffffffULL), + binary = binary_type, + bits = bits_type, + boolean = boolean_type, + decimal64 = decimal64_type, + empty = empty_type, + enumeration = enumeration_type, + identityref = identityref_type, + ['instance-identifier'] = instance_identifier_type, + leafref = leafref_type, + string = string_type, + union = union_type +} + -- Inline "grouping" into "uses". -- Inline "submodule" into "include". -- Inline "imports" into "module". @@ -609,33 +644,80 @@ function resolve(schema, features) node[prop] = nil return val end - local function push_env(node, env) - node = shallow_copy(node) - return node, {env=env, - groupings=pop_prop(node, 'groupings'), - typedefs=pop_prop(node, 'typedefs')} - end local function lookup(env, prop, name) if not env then error(prop..' not found: '..name) end if not env[prop] or not env[prop][name] then return lookup(env.env, prop, name) end - return env[prop][name], env + return env[prop][name] end - local visit - local function instantiate_grouping(env, name) - local grouping, grouping_env = lookup(env, 'groupings', name) - return visit(grouping, grouping_env) + local function lookup_lazy(env, prop, name) + local val = lookup(env, prop, name) + if type(val) == 'table' then return val end + -- Force lazy expansion and memoize result. + return val() end + local visit local function visit_top_level(node, env, prop) assert(not env[prop]) env[prop] = {} + local p = lookup(env, 'prefix', '_') for k,v in pairs(pop_prop(node, prop) or {}) do env[prop][k] = visit(v, env) + env[prop][p..':'..k] = env[prop][k] + end + end + local function visit_lazy(tab, env) + local ret = {} + local prefix = lookup(env, 'prefix', '_') + local function error_recursion() + error('mutually recursive typedefs or groupings') + end + for k,v in pairs(tab) do + -- FIXME: Only add prefix:k if at top level. + local function lazy() + ret[k] = error_recursion + ret[prefix..':'..k] = ret[k] + ret[k] = visit(v, env) + ret[prefix..':'..k] = ret[k] + return ret[k] + end + ret[k] = lazy + ret[prefix..':'..k] = ret[k] + end + return ret + end + function visit_type(node, env) + node = shallow_copy(node) + local success, typedef = pcall(lookup, env, 'typedefs', node.id) + if success then + -- Could be that typedef is still lazy. We didn't use + -- lookup_lazy because we don't want the pcall to hide errors + -- from the lazy expansion. + if type(typedef) == 'function' then typedef = typedef() end + node.base_type = typedef + else + -- If the type name wasn't bound, it must be primitive. + local make_primitive = assert(primitive_type_constructors[node.id], + 'unknown type: '..node.id) + node.primitive_type = make_primitive(node) end + return node end function visit(node, env) - node, env = push_env(node, env) + node = shallow_copy(node) + env = {env=env} + if node.typedefs then + -- Populate node.typedefs as a table of thunks that will + -- lazily expand and memoize their result when called. This + -- is not only a performance optimization but also allows the + -- typedefs to be mutually visible. + env.typedefs = visit_lazy(pop_prop(node, 'typedefs'), env) + end + if node.groupings then + -- Likewise expand groupings at their point of definition. + env.groupings = visit_lazy(pop_prop(node, 'groupings'), env) + end local when = pop_prop(node, 'when') if when then print('warning: assuming "when" condition to be true: '..when.value) @@ -650,10 +732,11 @@ function resolve(schema, features) return nil, env end end + if node.type then node.type = visit_type(node.type, env) end for k,v in pairs(node.body or {}) do if v.kind == 'uses' then -- Inline "grouping" into "uses". - local grouping = instantiate_grouping(env, v.id) + local grouping = lookup_lazy(env, 'groupings', v.id) node.body[k] = nil for k,v in pairs(grouping.body) do assert(not node.body[k], 'duplicate identifier: '..k) @@ -666,10 +749,10 @@ function resolve(schema, features) end return node, env end - local function merge_tables(dst, src) + local function include(dst, src) for k,v in pairs(src) do assert(dst[k] == nil or dst[k] == v, 'incompatible definitions: '..k) - dst[k] = v + if not k:match(':') then dst[k] = v end end end local linked = {} @@ -688,16 +771,17 @@ function resolve(schema, features) local submodule = lookup(env, 'submodules', k) assert(submodule.belongs_to.id == node.id) submodule, submodule_env = link(submodule, env) - merge_tables(module_env.extensions, submodule_env.extensions) - merge_tables(module_env.features, submodule_env.features) - merge_tables(module_env.identities, submodule_env.identities) - merge_tables(module_env.typedefs, submodule_env.typedefs) - merge_tables(module_env.groupings, submodule_env.groupings) - merge_tables(module_body, submodule.body) + include(module_env.extensions, submodule_env.extensions) + include(module_env.features, submodule_env.features) + include(module_env.identities, submodule_env.identities) + include(module_env.typedefs, submodule_env.typedefs) + include(module_env.groupings, submodule_env.groupings) + include(module_body, submodule.body) end if node.prefix then assert(node.kind == 'module', node.kind) module_env.prefixes[node.prefix] = node.namespace + module_env.prefix = {_=node.prefix} end for k,v in pairs(pop_prop(node, 'imports')) do assert(not module_env.prefixes[v.prefix], 'duplicate prefix') @@ -709,7 +793,9 @@ function resolve(schema, features) for _,prop in ipairs({'extensions', 'features', 'identities', 'typedefs', 'groupings'}) do for k,v in pairs(env[prop]) do - module_env[prop][prefix..':'..k] = v + if not k:match(':') then + module_env[prop][prefix..':'..k] = v + end end end end From c6f9d143897285f68859d24e810b7d56badcd10f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 26 Oct 2016 16:27:38 +0200 Subject: [PATCH 055/631] Add SnabbVMX manual --- src/program/snabbvmx/Makefile | 28 ++ src/program/snabbvmx/doc/.gitignore | 4 + .../snabbvmx/doc/README.configuration.md | 169 ++++++++ src/program/snabbvmx/doc/README.install.md | 96 +++++ src/program/snabbvmx/doc/README.md | 112 +++++ .../snabbvmx/doc/README.troubleshooting.md | 400 ++++++++++++++++++ src/program/snabbvmx/doc/README.userguide.md | 368 ++++++++++++++++ src/program/snabbvmx/doc/genbook.sh | 28 ++ 8 files changed, 1205 insertions(+) create mode 100644 src/program/snabbvmx/Makefile create mode 100644 src/program/snabbvmx/doc/.gitignore create mode 100644 src/program/snabbvmx/doc/README.configuration.md create mode 100644 src/program/snabbvmx/doc/README.install.md create mode 100644 src/program/snabbvmx/doc/README.md create mode 100644 src/program/snabbvmx/doc/README.troubleshooting.md create mode 100644 src/program/snabbvmx/doc/README.userguide.md create mode 100755 src/program/snabbvmx/doc/genbook.sh diff --git a/src/program/snabbvmx/Makefile b/src/program/snabbvmx/Makefile new file mode 100644 index 0000000000..978acc87c0 --- /dev/null +++ b/src/program/snabbvmx/Makefile @@ -0,0 +1,28 @@ +Q= @ +E= @echo + +snabb-lwaftr-doc: doc/snabb-lwaftr.pdf doc/snabb-lwaftr.html doc/snabb-lwaftr.epub + +doc/snabb-lwaftr.md: + (cd doc; ./genbook.sh) > $@ + +doc/snabb-lwaftr.pdf: doc/snabb-lwaftr.md + $(E) "PANDOC $@" + $(Q) (cd doc; pandoc -S --toc --chapters -o snabb-lwaftr.pdf snabb-lwaftr.md) + +doc/snabb-lwaftr.html: doc/snabb-lwaftr.md + $(E) "PANDOC $@" + $(Q) (cd doc; pandoc --self-contained --css="../../../doc/style.css" -S -s --toc --chapters -o snabb-lwaftr.html snabb-lwaftr.md) + +doc/snabb-lwaftr.epub: doc/snabb-lwaftr.md + $(E) "PANDOC $@" + $(Q) (cd doc; pandoc --self-contained --css="../../../doc/style.css" -S -s --toc --chapters -o snabb-lwaftr.epub snabb-lwaftr.md) + +CLEAN = doc/snabb-lwaftr.* + +clean: + $(E) "RM $(CLEAN)" + $(Q)-rm -rf $(CLEAN) + $(Q) (cd ../../; make clean) + +.PHONY: clean diff --git a/src/program/snabbvmx/doc/.gitignore b/src/program/snabbvmx/doc/.gitignore new file mode 100644 index 0000000000..41750185b6 --- /dev/null +++ b/src/program/snabbvmx/doc/.gitignore @@ -0,0 +1,4 @@ +snabb-lwaftr.epub +snabb-lwaftr.html +snabb-lwaftr.md +snabb-lwaftr.pdf diff --git a/src/program/snabbvmx/doc/README.configuration.md b/src/program/snabbvmx/doc/README.configuration.md new file mode 100644 index 0000000000..c2aa6ab12d --- /dev/null +++ b/src/program/snabbvmx/doc/README.configuration.md @@ -0,0 +1,169 @@ +# Configuration + +## SnabbVMX's configuration file + +SnabbVMX has its own configuration file (Example: **snabbvmx-lwaftr-xe0.cfg**). +Configuration file is actually a Lua file, that is processed by `snabbvmx lwaftr` +command. + +It contains a reference to a Snabb's **lwAFTR** configuration file (which +contains a reference to a binding-table). + +**snabbvmx-lwaftr-xe0.cfg** + +```lua +return { + lwaftr = "snabbvmx-lwaftr-xe0.conf", + ipv6_interface = { + cache_refresh_interval = 1, + mtu = 9500, + }, + ipv4_interface = { + ipv4_address = "10.0.1.1", + cache_refresh_interval = 1, + mtu = 1460, + }, + setup { + vlan = 421, + } +} +``` + +SnabbVMX defines extra configuration parameters for `ipv4_interface`/`ipv6_interface` +(deactivate/activate fragmentation, dynamic/static next-hop resolution, MTU). +In addition, it also allows a `setup` option for extra configuration. + +## Snabb's lwAFTR configuration files + +SnabbVMX's configuration file `lwaftr` attribute points out to a Snabb's lwAFTR +configuration file. This configuration file keeps a reference to a +binding-table, among other information, which is an important piece of a +lwAFTR deployment. + +Here is how a lwAFTR configuration file looks like: + +``` +binding_table = binding_table.txt, +vlan_tagging = false, +aftr_ipv6_ip = fc00::100, +aftr_mac_inet_side = 02:AA:AA:AA:AA:AA, +inet_mac = 02:99:99:99:99:99, +ipv6_mtu = 9500, +policy_icmpv6_incoming = DROP, +policy_icmpv6_outgoing = DROP, +icmpv6_rate_limiter_n_packets = 6e5, +icmpv6_rate_limiter_n_seconds = 2, +aftr_ipv4_ip = 10.0.1.1, +aftr_mac_b4_side = 02:AA:AA:AA:AA:AA, +next_hop6_mac = 02:99:99:99:99:99, +ipv4_mtu = 1460, +policy_icmpv4_incoming = DROP, +policy_icmpv4_outgoing = DROP, +``` + +And here is its referred binding-table: + +``` +psid_map { + 193.5.1.100 { psid_length=6, shift=10 } +} +br_addresses { + fc00::100 +} +softwires { + { ipv4=193.5.1.100, psid=1, b4=fc00:1:2:3:4:5:0:7e } + { ipv4=193.5.1.100, psid=2, b4=fc00:1:2:3:4:5:0:7f } + { ipv4=193.5.1.100, psid=3, b4=fc00:1:2:3:4:5:0:80 } + { ipv4=193.5.1.100, psid=4, b4=fc00:1:2:3:4:5:0:81 } + ... + { ipv4=193.5.1.100, psid=63, b4=fc00:1:2:3:4:5:0:bc } +} + +``` + +Some of the lwAFTR's configuration fields are of special relevance for +SnabbVMX. Although SnabbVMX can specify its own MTU and VLAN values, if those +attributes are defined in a lwAFTR configuration file they always take +precedence. + +Please refer to Snabb's lwAFTR documentation for a detailed description about +lwAFTR's configuration file and binding-table (Chapters 3 and 4). + +## Configuration examples + +### Dynamic next-hop resolution + +```lua +return { + lwaftr = "snabbvmx-lwaftr-xe0.conf", + ipv6_interface = { + cache_refresh_interval = 1, + mtu = 9500, + }, + ipv4_interface = { + ipv4_address = "10.0.1.1", + cache_refresh_interval = 1, + mtu = 1460, + }, +} +``` + +Parameters: + +- `ipv6_interface`: Configuration for lwAFTR's IPv6 interface (B4-side). +- `cache_refresh_interval`: Send next-hop resolution packet every second. +- `mtu`: Maximum Transfer Unit for IPv6 interface. + + +### Static next-hop resolution + +```lua +return { + lwaftr = "snabbvmx-lwaftr-xe0.conf", + ipv6_interface = { + mtu = 9500, + next_hop_mac = "02:aa:aa:aa:aa:aa", + fragmentation = false, + }, + ipv4_interface = { + ipv4_address = "10.0.1.1", + next_hop_mac = "02:99:99:99:99:99", + fragmentation = false, + mtu = 9500, + }, +} +``` + +Parameters: + +- `next_hop_mac`: MAC address of the nexthop. Outgoing IPv4 or IPv6 packets +will use this MAC address as Ethernet destination address. +- `fragmentation`: Boolean field. Indicated whether IPv4 or IPv6 packets get +fragmented by the lwAFTR in case packets are too big (larger than MTU size). + +### Additional setup + +``` +return { + ... + settings = { + vlan = 444, + ingress_drop_monitor = 'flush', + ingress_drop_threshhold = 100000, + ingress_drop_wait = 15, + ingress_drop_interval = 1e8, + } +} +``` + +Parameters: + +- `vlan`: Sets the same VLAN tag for IPv4 and IPv6. If lwAFTR's configuration +defines VLAN tags, they take prevalence. +- If `vlan_tag_v4` and `vlan_tag_v6` are defined in lwAFTR configuration, they +take prevalence. In that case, **SplitV4V6** app is not needed and two virtual +interfaces are initialized instead, one for IPv4 and another one for IPv6. +Each of them with its own VLAN tag. +- Ingress packet drop parameters initializes several features of the Snabb's +lwAFTR `ingress_drop_monitor` timer. A timer that periodically reports about +the NIC ingress packet drops. diff --git a/src/program/snabbvmx/doc/README.install.md b/src/program/snabbvmx/doc/README.install.md new file mode 100644 index 0000000000..7f36eae2e6 --- /dev/null +++ b/src/program/snabbvmx/doc/README.install.md @@ -0,0 +1,96 @@ +# Instalation + +## How to install it + +Snabb is a programming network toolkit as well as ready-to-use network +utilities suite. Every Snabb executable features several Snabb programs: + +```bash +$ sudo ./snabb +Usage: ./snabb ... + +This snabb executable has the following programs built in: + example_replay + example_spray + firehose + lisper + lwaftr + packetblaster + pci_bind + snabbmark + snabbnfv + snabbvmx + snsh + test + top +``` + +Type `snabb ` to run a specific program. Usually a simple program +call prints out its user help: + +```bash +$ sudo ./snabb snabbvmx +Usage: + snabbvmx check + snabbvmx lwaftr + snabbvmx query + snabbvmx top + +Use --help for per-command usage. + +Example: + snabbvmx lwaftr --help +``` + +There is no specific script to install a snabb executable. Once it's built, +a snabb executable includes all its dependencies, including a LuaJIT interpreter, +in a single binary. Thus, it's possible to relocate a snabb executable to any +folder in the system. Move it to folder in PATH, so the system can alway locate it: + +```bash +$ echo $PATH +/home/user/bin:/opt/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin: +/usr/bin:/sbin:/bin + +$ cp snabb /opt/local/bin +``` + +When snabb executable is renamed to one of its featured programs, it will always +the program is named after. For instance, to always run snabbvmx simply rename +snabb to snabbvmx. + +```bash +$ mv snabb snabbvmx +$ sudo ./snabbvmx +Usage: + snabbvmx check + snabbvmx lwaftr + snabbvmx query + snabbvmx top +``` + +## SnabbVMX tools + +SnabbVMX program (**program/snabbvmx/**) features a series of subcommands or tools: + +- **check**: Used to verify the correctness of the lwAFTR logic. +- **nexthop**: Used to retrieve the nexthop cached values (available in shared memory). +- **query**: Used to check out the counter values of a running SnabbVMX instance. +- **lwaftr**: Main program. Set ups SnabbVMX network design and runs it. +- **top**: Similar to Snabb's top. Prints out Gb and Mpps in IPv4 and IPv6 interfaces. +Includes reports about counters and ingress-packet-drops. + +There is an additional program in snabb called **packetblaster**. Packetblaster +includes a *lwaftr* mode. This mode is very useful to generate live traffic matching +a binding-table. + +```bash + $ ./snabb packetblaster lwaftr --src_mac 02:02:02:02:02:02 \ + --dst_mac 02:42:df:27:05:00 \ + --b4 2001:db8::40,10.10.0.0,1024 \ + --aftr 2001:db8:ffff::100 \ + --count 60001 --rate 3.1 --pci 0000:05:00.1 +``` + +Please check the [How to use it?](README.userguide.md) chapter for a more +detailed view on each tool. diff --git a/src/program/snabbvmx/doc/README.md b/src/program/snabbvmx/doc/README.md new file mode 100644 index 0000000000..ca3f017f2a --- /dev/null +++ b/src/program/snabbvmx/doc/README.md @@ -0,0 +1,112 @@ +# SnabbVMX + +## What's SnabbVMX? + +SnabbVMX is a network design that combines Snabb's lwAFTR app and a VM managed +by Snabb. For a detailed explanation of what's an lwAFTR, please refer to +lwAFTR docs. + +Snabb's lwAFTR app, in addition to managing lw4o6 traffic (IPv4 and IPv6 packets), +it also manages non-lwAFTR traffic such as pings to lwAFTR interfaces, ARP and +NDP resolution. This traffic is managed by addition applications connected to +the lwAFTR network design in program/lwaftr/run.lua. Management of this type +of traffic is a requiremennt in RFC 7596. + +SnabbVMX uses a different approach. It setups a network design where only lw4o6 +packets are managed by the lwAFTR app, while everything else is forwarded to a VM +which is in charge of providing ping, ARP and NDP resolution. + +## App network + +SnabbVMX's app network: + +![SnabbVMX](.images/snabbvmx.png) + +SnabbVMX works in one single 10Gb NIC. Incoming traffic gets to an Intel82599 app +managed by Snabb. + +App V4V6 splits traffic between IPv4 or IPv6 and forwards it to the correspondent +Reassembler app. Let's take IPv4 traffic flow as example. ReassemblerV4 forwards +packets to a Next Hop Forwarder app. This app decides whether to forward traffic +to the lwAFTR app or to the VM, based on unicast/broadcast destination MAC, +matching local IPv4 destination or IPv4-in-IPv6. + +App VhostUser communicates with a VM run by QEMU using the VhostUser network +backend available in QEMU. Communication between both processes happens via a +socket. + +App V4V6, in combination with a Tap app, allows monitoring of packets coming +in and out of the physical port based on a matching IPv4 address, either as +source, destination or within an IPv4-in-IPv6 packet. + +#### How to run SnabbVMX + +``` + $ sudo ./snabb snabbvmx lwaftr --id SnabbVMX1 \ + --conf snabbvmx-lwaftr.cfg \ + --pci 81:00.0 \ + --mac 02:AA:AA:AA:AA:AA \ + --sock /tmp/vhuser.sock +``` + +#### Configuration file + +`snabbvmx-lwaftr.cfg` + +``` +return { + lwaftr = "snabbvmx-lwaftr.conf", + ipv6_interface = { + cache_refresh_interval = 1, + }, + ipv4_interface = { + ipv4_address = "10.0.1.1", + cache_refresh_interval = 1, + }, + settings = { + vlan = false, + }, +} +``` + +lwaftr points to Snabb's lwAFTR configuration file. + +Othe attributes are further refined by SnabbVMX. Attributes `ipv6_interface` +and `ipv4_interface` are mandatory and ipv4_interface must include the IP +addresses of the correspondent lwAFTR IPv4 interfaces. The IPv4 address is +used to send matching packets to the VMX instead of trying to find a match in +the binding table for encap. + +Attribute `settings` may include a `vlan` tag attribute, which can be either +false in case VLAN tagging is not enabled or a VLAN tag number (0-4095). + +Reassembly and fragmentation are deactivated by default in SnabbVMX. In case +they are needed, set `ipv6_interface.fragmentation = true`. + +Interfaces also allow setting of IPv4 and IPv6 ingress/egree filters. The +filters are expessed as a packet-filtering expression (like tcpdump) or point +to a file containing a pf expression. + +#### Components overview + +List of apps used in SnabbVMX network design: + +* **V4V6** (`apps/lwaftr/V4V6.lua`): Categorizes traffic as IPv4 and IPv6 and +forwards it to the correspondent link. +* **Tap** (`apps/tap/tap.lua`): Used to monitor traffic in V4V6 via a Tap +interface. Matching IPv4 address is controlled by "lwaftr snabbvmx monitor". +* **ReassemblerV6**/**FragmenterV6** (`apps/lwaftr/ipv6_apps.lua`): IPv6 reassembler and fragmenter apps. +* **ReassemblerV4**/**FragmenterV4** (`apps/lwaftr/ipv4_apps.lua`): IPv6 reassembler and +fragmenter apps. +* **Next-hop Forwarder** (`apps/nh_fwd/nh_fwd.lua`): It contains to forwarding apps: +nh_fwd6 and nh_fwd4. The apps forwards incoming traffic to a service, in this +case to a lwAFTR app, and to a VM. Routing decisions are based on next-hop +MAC address learned from the VM and by IP destination address of incoming +packets. +* **lwAFTR** (`apps/lwaftr/lwaftr.lua`): Implements a lwAFTR component as specified +in RFC7569. +* **VhostUser** (`apps/vhost/vhost_user.lua`): A driver complying Virtio which allows +a program in user space, in this case Snabb, to act as an hypervisor of the +networking side of a guest. Similarly, the guest side has to be managed by a +VhostUser network backend (already built-in in QEMU). + diff --git a/src/program/snabbvmx/doc/README.troubleshooting.md b/src/program/snabbvmx/doc/README.troubleshooting.md new file mode 100644 index 0000000000..34e9e41158 --- /dev/null +++ b/src/program/snabbvmx/doc/README.troubleshooting.md @@ -0,0 +1,400 @@ +# Testing & troubleshooting + +## Troubleshooting + +### Troubleshooting in the lwAFTR + +If traffic managed by the lwAFTR component is not responding as expected: packets +that should get decapsulate don't match a softwire, some packets are dropped, +traffic don't get in, etc, the most convenient is to inspect lwAFTR counters +to diagnose the problem. Snabb's `lwaftr query` tool can be used to obtain +the counter values of a lwAFTR instance. + +Snabb's lwAFTR manual includes a chapter covering troubleshooting and counters, +with charts about the most common lwAFTR paths (encapsulation, decapsulation, +hairpinning, etc). Please refer to that guide for troubleshooting in the +lwAFTR logic (Chapter 8 - Counters). + +The manual also includes a section covering troubleshooting related with running +a lwAFTR instance. This information can be useful in case of running the lwAFTR +as a standalone application and not via SnabbVMX. + +### Troubleshooting in SnabbVMX + +This section covers most common problems when running a SnabbVMX instance. + +#### Cannot start SnabbVMX instance + +`Description`: When running a snabbvmx instance an error reporting `failed to lock + + [C]: in function 'error' + core/main.lua:26: in function 'assert' + lib/hardware/pci.lua:143: in function 'map_pci_memory_locked' +``` + +`Solution`: This error happens when trying to run a Snabb program, in this case +SnabbVMX, on a NIC which is already in use. Please check that there's no other +Snabb instance running on the same NIC (in this example, `0000:81:00.0`). + +#### SnabbVMX running but not receiving traffic + +`Description`: `snabbvmx lwaftr` instance is running on a NIC but `snabbvmx top` +reports the SnabbVMX instance is receiving no traffic. + +`Solution`: This is a common problem which might be originated by various causes. + +1. If no traffic is received at all, the most likely cause is that the selected +lwAFTR binding table doesn't contain any valid softwire for incoming traffic. + +2. Another possible cause is that SnabbVMX is running with dynamic nexthop +resolution, but dynamic nexthop resolution is not working. + +#### Dynamic nexthop resolution is not working + +`Description`: SnabbVMX is running with dynamic nexthop resolution but no traffic +is leaving the lwAFTR. + +`Solution`: Check SnabbVMX is indeed running with dynamic nexthop resolution. +SnabbVMX's configuration file should have attribute `cache_refresh_interval` set to a +value higher than 0. + +```bash +ipv6_interface = { + cache_refresh_interval = 1, +}, +ipv4_interface = { + ipv4_address = "10.0.1.1", + cache_refresh_interval = 1, +}, +``` + +If that's correct, the likely caused is that refreshment packets are not arriving +to the VM. A refreshment packet is a packet that is sent periodically to the VM +to trigger nexthop cache resolution. + +Packets might not be arriving to the VM because there's no VM actually running. +Was SnabbVMX started with a sock address? + +```bash +sudo ./snabb snabbvmx lwaftr --id xe0 --conf snabbvmx-lwaftr-xe0.cfg + --pci 02:00.0 --mac 02:aa:aa:aa:aa:aa --sock /tmp/vh1a.sock +``` + +If that's correct, another cause is that the selected MAC address (02:aa:aa: +aa:aa:aa, in the example above), doesn't match the lwAFTR's configuration +attributes `aftr_mac_inet_side` and `aftr_mac_b4_side`. Check next section at +the bottom for more details about that. + +#### SnabbVMX doesn't respond to IPv4 or IPV6 pings + +`Description`: Trying to ping SnabbVMX on one of its IPv4 or IPv6 interfaces gets +no response. + +`Solution`: Double-check the target address is indeed lwAFTR IPV4 or IPv6 +addresses. Steps: + +1. Open the selected SnabbVMX configuration file and go to its lwAFTR +configuration. +2. Check the values of attributes `aftr_ipv6_ip` and `aftr_ipv4_ip` are +equals to the target address. + +In case the address is correct, double-check SnabbVMX MAC address is the same +as `aftr_mac_inet_side` and `aftr_mac_b4_side`. In SnabbVMX these values must +be the same, as there's a single NIC for both interfaces. + +SnabbVMX should have been initiated with this MAC address value. Search a +SnabbVMX process to see its command line parameters: + +```bash +$ ps aux | grep snabbvmx +root 16115 0.0 0.0 133856 2936 pts/2 S+ 22:39 0:00 + sudo ./snabb snabbvmx lwaftr --conf snabbvmx-lwaftr.cfg --id xe1 + --pci 0000:81:00.0 --mac 02:AA:AA:AA:AA:AA +``` + +#### SnabbVMX decapsulation path works but not encapsulation + +`Description`: `snabbvmx top` reports running SnabbVMX instance is able to +decapsulate packets but not to encapsulate. + +lwaftr (rx/tx/txdrop in Mpps) rx tx rxGb txGb txdrop +lwaftr_v6 1.53 0 4.80 0 0.000022 +lwaftr_v4 0 1.53 0 4.31 0 + +`Solution`: If the decapsulation path works that means lwAFTR is able to +decapsulate IPv6 packets coming from the B4. That means the IPv4 source address +and source port of the IPv6 encapsulated packet matches a softwire in the +binding table. However, when a packet arrives to the lwAFTR from the Internet +the packet is dropped, thus RX is 0 in lwaftr_v4. + +The destination address and port of an incoming IPv4 packet should match a +softwire in the binding table. What's the destination address and port of +incoming packets? Is the lwAFTR using a VLAN tag for its IPv4 interface? Most +likely incoming packets are not VLAN tagged. Check if SnabbVMX's config file +has a VLAN tag set. Check also if the referred lwAFTR config file has a VLAN +tag set. + +What's the MTU size of the IPv4 interface? Check that value in SnabbVMX and +lwAFTR configuration file. If the MTU size is too small, packets will be +fragmented. A extremely small MTU size and fragmentation disable will cause +most of the packets to be dropped. + +#### Packets don't get mirrored + +`Description`: You're using a tap interface for monitoring lwAFTR packets but +nothing gets in. + +`Solution`: + +1. Check you're running SnabbVMX with mirroring enabled. +2. Check you're running `lwaftr monitor` pointing to the IPv4 address you would +like to monitor. For testing purposes, set `lwaftr monitor` to `all`. +3. Check your tap interface is up. + +--- + +## Tests overview + +Test are useful to detect bugs in the `snabbvmx` and double-check everything +is working as expected. + +SnabbVMX features three types of tests: + +* Lua selftests: Unit tests for several modules +(**apps/lwaftr/V4V6.lua** and **apps/lwaftr/nh_fwd.lua**). +* Bash selftest: Complex setups to test certain functionality (**program/ +snabbvmx/tests/selftest.sh** and **program/snabbvmx/tests/nexthop/ +selftest.sh**). +* End-to-end tests: Particular end-to-end test cases for SnabbVMX +(**program/snabbvmx/tests/end-to-end/selftest.sh**). + +Usually Lua selftests and Bash selftest won't fail as these test must successfully +pass on every Snabb deliverable. End-to-end tests won't fail either, but it is +interesting to learn how to add new SnabbVMX's end-to-end tests to diagnose +potential bugs. + +All these tests are run by Snabb's Continuous Integration subsystem (snabb-bot). +The test can also be run when executing **make test** (only if Snabb's source +is available). + +```bash +$ sudo make test +TEST apps.lwaftr.V4V6 +TEST apps.lwaftr.nh_fwd +... +TEST program/snabbvmx/tests/selftest.sh +SKIPPED testlog/program.snabbvmx.tests.selftest.sh +TEST program/snabbvmx/tests/nexthop/selftest.sh +SKIPPED testlog/program.snabbvmx.tests.nexthop.selftest.sh +... +TEST program/snabbvmx/tests/end-to-end/selftest.sh +... +TEST program/lwaftr/tests/end-to-end/selftest.sh +TEST program/lwaftr/tests/soaktest/selftest.sh +``` +Execution of a test can return 3 values: + +* **TEST**: The test run successfully (exit code 0) +* **SKIPPED**: The test was skipped, usually because it needs to access to +a physical NIC and its PCI address was not setup (exit code 43). +* **ERROR**: Test test finished unexpectedly (exit code > 0 and != 43). + +### Lua selftests + +* **apps/lwaftr/V4V6.lua**: Builds customized IPv4 and IPv4-in-IPv6 packets +and joins the packets to a single link (*test_join*) or splits the packets +to two different links (*test_split*). +* **apps/lwaftr/nh_fwd.lua**: Builds customized packets and test the 3 code +paths of next-hop forwarder: *from-wire-to-{lwaftr, vm}*, +*from-vm-to-{lwaftr, wire}*, *from-lwaftr-to-{vm, wire}*. + +To run an individual Lua module (app, program or library) selftest: + +``` +$ sudo ./snabb snsh -t apps.lwaftr.V4V6 +V4V6: selftest +OK +``` + +### Bash selftests + +* **program/snabbvmx/tests/selftest.sh**: Tests Ping, ARP and NDP resolution +by the VM. +* **program/snabbvmx/tests/nexthop/selfttest.sh**: Tests nexthop resolution +by the VM. + +For instance, the result of executing `snabbvmx/tests/selftest.sh` would be the +following: + +```bash +$ sudo SNABB_PCI0=83:00.0 SNABB_PCI1=03:00.0 \ + program/snabbvmx/tests/selftest.sh + +Launch Snabbvmx +Waiting for VM listening on telnet port 5000 to get ready... [OK] +Ping to lwAFTR inet side: OK +Ping to lwAFTR inet side (Good VLAN): OK +Ping to lwAFTR inet side (Bad VLAN): OK +Ping to lwAFTR B4 side: OK +Ping to lwAFTR B4 side (Good VLAN): OK +Ping to lwAFTR B4 side (Bad VLAN): OK +ARP request to lwAFTR: OK +ARP request to lwAFTR (Good VLAN): OK +ARP request to lwAFTR (Bad VLAN): OK +NDP request to lwAFTR: OK +NDP request to lwAFTR (Good VLAN): OK +NDP request to lwAFTR (Bad VLAN): OK +``` + +NOTE: To successfully run the test `SNABB_PCI0` and `SNABB_PCI1` cards must +be wired together. + +The test goes through several steps: + +1. Run **SnabbVMX** on NIC **SNABB_PCI0**. +2. Run **QEMU**. +3. Configure VM eth0 interface with MAC, IPv4/IPv6 address, ARP & NDP cache table. +4. Runs **tcpreplay** on NIC SNABB_PCI1. Sample packets reach the VM. +5. Outgoing packets from the VM are mirrored to a tap interface (tap0). +6. Captures **responses on tap0** and compares them with expected results. + +The input data as well as the expected output is at `program/snabbvmx/tests/pcap`. + +The test validated VLAN packets too. However, there are not VLAN tagged versions +of the expected output. The reason is that it is the NIC which tags and untags +a packet. Since the packet have not leave the NIC yet, they come out from the VM +untagged. + +```bash +$ tcpdump -qns 0 -ter good/arp-request-to-lwAFTR.pcap +reading from file arp-request-to-lwAFTR.pcap, link-type EN10MB (Ethernet) +52:54:00:00:00:01 > ff:ff:ff:ff:ff:ff, 802.1Q, length 46: vlan 333, p 0, +ethertype ARP, Request who-has 10.0.1.1 tell 10.0.1.100, length 28 +``` + +The other bash selftest, validates correct resolution of the nexthop by the VM. + +```bash +$ sudo SNABB_PCI0=03:00.0 SNABB_PCI1=83:00.0 \ + program/snabbvmx/tests/nexthop/selftest.sh +Waiting for VM listening on telnet port 5000 to get ready... [OK] +Resolved MAC inet side: 6a:34:99:99:99:99 [OK] +Resolved MAC inet side: 4f:12:99:99:99:99 [OK] +``` + +The test goes through several steps: + +1. Run SnabbVMX on SNABB_PCI0. +2. Run VM. +3. Send packets in loop to SNABB_PCI1 during 10 seconds. Packets reach the VM. + +```bash +$ packetblaster replay -D 10 $PCAP_INPUT/v4v6-256.pcap $SNABB_PCI1 +``` + +4. Retrieve out nexthop values (snabbvmx nexthop). +5. Compare to expected values. +6. Timeout if 10 seconds elapsed. + +NOTE: Currently the test is not working correctly. The returned MAC should be +`02:99:99:99:99:99`. + +### End-to-end tests + +**program/snabbvmx/tests/end-to-end/selftest.sh**: Runs end-to-end tests +(normal and VLAN). + +The end-to-end tests is a test suite that tests the correctness of the lwAFTR +logic. + +Files in **program/snabbvmx/tests/end-to-end/**: + +* **test_env.sh**: Contains the test cases. +* **core-end-to-end.sh**: Actually runs the test cases using **snabbvmx check**. +* **data/**: Directory containing sample pcap input, expected pcap output, +configuration files and binding tables. +* **end-to-end.sh**: Runs core-end-to-end.sh on normal packets. +* **end-to-end-vlan.sh**: Runs core-end-to-end.sh on VLAN packets. +* **selftest.sh**: Runs both end-to-end.sh and end-to-end-vlan.sh + +To run SnabbVMX's end-to-end test suite: + +```bash +$ sudo ./end-to-end.sh +Testing: IPv6 fragments and fragmentation is off +done +Test passed +All end-to-end lwAFTR tests passed. +``` + +The end-to-end test suite relies on `snabbvmx check` to run each test. + + +```bash +$ sudo ./snabb snabbvmx check +Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP + [COUNTERS.LUA] +``` + +Parameters: + +- **CONF**: SnabbVMX configuration file. +- **V4-IN.PCAP**: Incoming IPv4 packets (from inet). +- **V6-IN.PCAP**: Incoming IPv6 packets (from b4). +- **V4-OUT.PCAP**: Outgoing IPv4 packets (to inet, decapsulate). +- **V6-OUT.PCAP**: Outgoing IPv6 packets (to b4, encapsulated) +- **[COUNTERS.LUA]**: Lua file with counter values. + +Although SnabbVMX works in one single interface, snabbvmx check requires that +the packet split is already done and provides an split output too. + +Snabb's lwAFTR includes an end-to-end test suite counterpart. In most cases, +the lwAFTR's correctness will be test via Snabb's lwAFTR end-to-end tests. +However, since SnabbVMX uses a different configuration file, the network design +that it brings up might be slightly different than Snabb's lwAFTR. For instance, +Snabb's lwAFTR fragmentation is always activated while in SnabbVMX is an optional +argument, either for IPV4 and IPv6 interfaces. + +Modifications in the app chain might run executed lwAFTR's data plane in a different +way, bringing up conditions that were not covered by lwAFTR's data-plane. For +this reason, a specific end-to-end test suite was added to SnabbVMX. A brand-new +test set that covers specifics needs of SnabbVMX. If a bug is found most likely +its resolution will happen in Snabb's lwAFTR code, resulting into the addition of +a new test in lwAFTR's test-suite. + +## How to write a SnabbVMX end-to-end test + +Imagine you have detected an error in the lwAFTR. The first step is to obtain +the configuration file that SnabbVMX was using, as well as a copy of lwAFTR's +configuration and binding table. With that information and knowing the error +report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get + decapsulate, etc), you craft a hand-made packet that meets the testing case. + +Now we can check what the lwAFTR would produce: + +```bash +sudo ./snabb snabbvmx -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap empty.pcap \ + /tmp/outv4.pcap /tmp/outv6.pcap counters.lua +``` + +The flag `-r` generates a counters file. + +Check your output matches what you expected: + +```bash +$ tcpdump -qns 0 -ter empty.pcap +reading from file empty.pcap, link-type EN10MB (Ethernet) +``` + +Checking what values are in the counters can give you a hint about whether +things are working correctly or not. + +Tip: Packets always arrive only in one interface, but the output might be +empty for both interfaces, non-empty and empty or non-empty for both cases. diff --git a/src/program/snabbvmx/doc/README.userguide.md b/src/program/snabbvmx/doc/README.userguide.md new file mode 100644 index 0000000000..c0c5cc2a33 --- /dev/null +++ b/src/program/snabbvmx/doc/README.userguide.md @@ -0,0 +1,368 @@ +# User guide + +The following chapter teaches you how to get SnabbVMX up-and-running. This +chapter includes references to concepts and terms not covered yet. Please +refer to other sections in this manual in case of doubt. + +## SnabbVMX lwaftr + +`snabbvmx lwaftr` is the main program. It setups the app network design and pass + it to Snabb's engine to run it. + +```bash +$ sudo ./snabb snabbvmx lwaftr +$ sudo ./snabb snabbvmx lwaftr | head -12 +Usage: lwaftr --help + +lwaftr --conf --id --pci --mac \ + --sock [-D ] [-v] + +Arguments: + + --conf configuration file for lwaftr service + --id port_id for virtio socket + --pci PCI device number for NIC (or Linux interface name) + --mac Ethernet address of virtio interface + --sock Socket path for virtio-user interfaces +``` + +The example below use static next-hop resolution so a VM is not needed. This +setup is useful to test a lwAFTR data-plane. + +**snabbvmx-lwaftr-xe0.cfg** + +```lua +return { + lwaftr = "snabbvmx-lwaftr-xe0.conf", + ipv6_interface = { + cache_refresh_interval = 1, + mtu = 9500, + }, + ipv4_interface = { + ipv4_address = "10.0.1.1", + cache_refresh_interval = 1, + mtu = 1460, + }, + settings = { + vlan = false, + ingress_drop_monitor = 'flush', + ingress_drop_threshhold = 100000, + ingress_drop_wait = 15, + ingress_drop_interval = 1e8, + }, +} +``` + +**snabbvmx-lwaftr-xe.conf** + +``` +binding_table = binding_table.txt.s, +aftr_ipv4_ip = 172.20.1.16, +aftr_ipv6_ip = 2001:db8::1, +aftr_mac_b4_side = 02:42:df:27:05:00, +aftr_mac_inet_side = 02:42:df:27:05:00, +inet_mac = 02:02:02:02:02:02, +ipv4_mtu = 9000, +ipv6_mtu = 9000, +next_hop6_mac = 02:02:02:02:02:02, +vlan_tagging = false, +``` + +**binding_table.txt.s** + +``` +psid_map { + 193.5.1.100 { psid_length=6, shift=10 } +} +br_addresses { + fc00::100 +} +softwires { + { ipv4=193.5.1.100, psid=1, b4=fc00:1:2:3:4:5:0:7e } + { ipv4=193.5.1.100, psid=2, b4=fc00:1:2:3:4:5:0:7f } + { ipv4=193.5.1.100, psid=3, b4=fc00:1:2:3:4:5:0:80 } + { ipv4=193.5.1.100, psid=4, b4=fc00:1:2:3:4:5:0:81 } + ... + { ipv4=193.5.1.100, psid=63, b4=fc00:1:2:3:4:5:0:bc } +} +``` + +Now we are ready to run SnabbVMX: + +```bash +$ sudo ./snabb snabbvmx lwaftr --id xe0 --conf snabbvmx-lwaftr-xe0.cfg \ + --pci 82:00.0 --mac 02:42:df:27:05:00 +Ring buffer size set to 2048 +loading compiled binding table from ./binding_table_60k.txt.s.o +compiled binding table ./binding_table_60k.txt.s.o is up to date. +Hairpinning: yes +nic_xe0 ether 02:42:df:27:05:00 +IPv6 fragmentation and reassembly: no +IPv4 fragmentation and reassembly: no +lwAFTR service: enabled +Running without VM (no vHostUser sock_path set) +nh_fwd6: cache_refresh_interval set to 1 seconds +loading compiled binding table from ./binding_table_60k.txt.s.o +compiled binding table ./binding_table_60k.txt.s.o is up to date. +nh_fwd4: cache_refresh_interval set to 1 seconds +Ingress drop monitor: flush (threshold: 100000 packets; + wait: 15 seconds; interval: 0.01 seconds) +``` + +Now we should send valid lwAFTR traffic to SnabbVMX. One way of doing it is +using a Snabb tool called **packetblaster**. Packetblaster has a lwaftr mode +that generates valid lwAFTR packets for a given binding-table configuration. + +```bash +$ sudo ./snabb packetblaster lwaftr --src_mac 02:02:02:02:02:02 \ + --dst_mac 02:42:df:27:05:00 --b4 2001:db8::40,10.10.0.0,1024 \ + --aftr 2001:db8:ffff::100 --count 60001 --rate 3.1 \ + --pci 0000:02:00.0 +``` + +## SnabbVMX top + +`snabbvmx top` prints out information of a running SnabbVMX instance. There might +be more than one Snabb process running: + +```bash +$ sudo ./snabb top +Multiple Snabb instances found. Select one: +11963 +11958 +``` + +Which one is my SnabbVMX process? Luckily, SnabbVMX top allows to query per 'id': + +```bash +$ sudo ./snabb snabbvmx top xe0 +lwaftr (rx/tx/txdrop in Mpps) rx tx rxGb txGb txdrop +lwaftr_v6 1.53 1.53 4.80 4.80 0.000022 +lwaftr_v4 1.53 1.53 4.31 4.31 0.000022 + +lwaftr_v6 in-ipv6-packets 211,255,481 1,526,560 +lwaftr_v6 in-ipv6-bytes 83,036,588,870 600,074,786 +lwaftr_v6 out-ipv6-packets 211,255,393 1,526,538 +lwaftr_v6 out-ipv6-bytes 83,036,242,914 600,061,830 +lwaftr_v6 drop-all-ipv6-iface-packets 3,537 22 +lwaftr_v6 in-ipv6-frag-reassembled 0 0 +lwaftr_v6 drop-ipv6-frag-invalid-reassembly 0 0 +lwaftr_v6 out-ipv6-frag 0 0 +lwaftr_v6 out-ipv6-frag-not 0 0 + + Total per second +nic ifInDiscards 1,547,333 47,449 +``` + +1.53 MPPS in both interfaces rx/tx. We are running at line-rate, although there + are packet drops. + +## SnabbVMX query + +SnabbVMX query prints out all the counters of a SnabbVMX instance in XML format: + +```xml + + + 0 + xe0 + 11958 + 00:00:00:00:00:00 + 00:00:00:00:00:00 + 0.0.0.0 + + 67198200 + 1 + 4663620304544 + 569220658720 + 1525764372 + + + ... + + + + 4488858 + 12721 + 4997698 + 269348948180 + 0 + 0 + 0 + + + + + 1476276089 + 0 + 0 + 0 + 0 + 0 + + + + +``` + +Useful for interacting with other systems in the network. + +Snabb's lwAFTR features a query tool too. This tool is aimed to query lwAFTR's +counters. Snabb's `lwaftr query` is covered in the last section of this chapter. + +## SnabbVMX check + +`snabbvmx check` is an utility tool use to validate lwAFTR correctnes. +SnabbVMX has its own version of it. + +```bash +$ sudo ./snabb snabbvmx check +Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP +[COUNTERS.LUA] +``` + +Using check is the step prior to add a new test in the end-to-end test suite. + +**CONF** is a SnabbVMX configuration file and **V4-IN** and **V6-IN** are the +incoming V4 and V6 interfaces. The tools reads packets from a pcap file. **V4-OUT** +and **V6-OUT** are the resulting packets after the lwAFTR processing. How the +packets are processed depends on the configuration file and the binding table. + +Although SnabbVMX works in one single interface, snabbvmx check requires that +the packet split is already done and provides an split output too. + +Imagine you have detected an error in the lwAFTR. First step is to obtain the +configuration that SnabbVMX was using, as well as a copy of lwAFTR's +configuration and binding table. With that information and knowing the error +report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get +decapsulate), you hand-made a packet that meets the testing case. Now we can +check what the lwAFTR will produce: + +```bash +sudo ./snabb snabbvmx -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap empty.pcap \ + /tmp/outv4.pcap /tmp/outv6.pcap counters.lua +``` + +The flag -r generates a counters file. + +Check your output matches what you expected: + +```bash +$ tcpdump -qns 0 -ter empty.pcap +reading from file empty.pcap, link-type EN10MB (Ethernet) +``` + +Checking what values are in counters can tell you if things are working +correctly or not. + +Tip: Packets always arrive only in one interface, but the output might be +empty for both interfaces, non-empty and empty or non-empty for both cases. + +## Other related tools + +### Snabb's lwaftr query + +Snabb's `lwaftr query` command can be used to print out counter's values of +a running lwAFTR instance (that also includes a SnabbVMX instance). In case of +query a SnabbVMX instance, the instance can be referred by its `id`. + +Counters are useful to debug and understand what data-paths are being taken. +Running `snabb lwaftr query ` lists all non-zero counters of a lwAFTR's +instance: + +```bash +$ sudo ./snabb lwaftr query xe0 +lwAFTR operational counters (non-zero) +drop-all-ipv4-iface-bytes: 7,642,666 +drop-all-ipv4-iface-packets: 21,653 +drop-all-ipv6-iface-bytes: 8,508,786 +drop-all-ipv6-iface-packets: 21,653 +drop-no-dest-softwire-ipv4-bytes: 7,642,666 +drop-no-dest-softwire-ipv4-packets: 21,653 +drop-no-source-softwire-ipv6-bytes: 8,508,786 +drop-no-source-softwire-ipv6-packets: 21,653 +in-ipv4-bytes: 458,034,561,846 +in-ipv4-packets: 1,297,289,752 +in-ipv6-bytes: 509,922,361,858 +in-ipv6-packets: 1,297,275,296 +ingress-packet-drops: 5,849,182 +out-icmpv4-bytes: 6,104,782 +out-icmpv4-packets: 21,653 +out-icmpv6-bytes: 8,942,294 +out-icmpv6-packets: 21,653 +out-ipv4-bytes: 458,029,812,134 +out-ipv4-packets: 1,297,275,296 +out-ipv6-bytes: 509,917,643,140 +out-ipv6-packets: 1,297,268,099 +``` + +It is possible to pass a filter expression to query only matching counters. +The example below prints out only outgoing related counters. + +```bash +$ sudo ./snabb lwaftr query 11958 out +lwAFTR operational counters (non-zero) +out-icmpv4-bytes: 7,460,356 +out-icmpv4-packets: 26,460 +out-icmpv6-bytes: 10,928,022 +out-icmpv6-packets: 26,460 +out-ipv4-bytes: 559,187,260,900 +out-ipv4-packets: 1,583,779,666 +out-ipv6-bytes: 622,534,239,098 +out-ipv6-packets: 1,583,770,321 +``` + +### Snabb's lwaftr monitor + +`lwaftr monitor` is a tool that helps monitorizing incoming and outgoing packets +in a lwAFTR. This feature must be combined with running SnabbVMX with mirroring +enabled. + +```bash +$ sudo ./snabb snabbvmx lwaftr --id xe0 --conf snabbvmx-lwaftr-xe0.cfg \ + --pci 82:00.0 --mac 02:42:df:27:05:00 --mirror tap0 +``` + +`mirror` parameter expects to find a tap interface. The interface must be up +in order to receive packets: + +```bash +$ sudo ip link set dev tap0 up +``` + +`lwaftr monitor` is set with a target IPv4 address. All lwAFTR packets which +IPv4 source match the target address will be mirrored in the tap interface. +In addition, monitor can be set with two special values: + +- `all`: mirrors all packets to the tap interface. +- `none`: mirrors nothing to the tap interface. + +### Snabb's packetblaster lwaftr + +`snabb packetblaster` is a built-in Snabb utility which can blast packets to a +NIC very fast. It features several modes: `synth`, `replay` and `lwaftr`. + +The `lwaftr` mode is specially suited to generate IPv4 and IPv6 packets that +match a binding table. Example: + +```bash +$ sudo ./snabb packetblaster lwaftr --src_mac 02:02:02:02:02:02 \ + --dst_mac 02:42:df:27:05:00 --b4 2001:db8::40,10.10.0.0,1024 \ + --aftr 2001:db8:ffff::100 --count 60001 --rate 3.1 \ + --pci 0000:02:00.0 +``` + +Parameters: + +- `src_mac`: Source MAC-Address. +- `dst_mac`: Destination MAC-Address. Must match a lwAFTR configuration file +MAC addresses. +- `b4`: IPV6,IPV4,PORT. IPV6, IPV4 and PORT values of the first softwire. +- `aftr`: IPv6 address of lwaftr server. Only one value can be specified. +- `count`: Number of B4 clients to simulate. +- `rate`: Rate in MPPS for the generated traffic. +- `pci`: Interface PCI address. + +These are the most essential parameters to generate valid lwAFTR traffic to +a SnabbVMX instance. `packetblaster lwaftr` features much more flags and +options. Please refer to its Snabb's manual for a more detailed description. diff --git a/src/program/snabbvmx/doc/genbook.sh b/src/program/snabbvmx/doc/genbook.sh new file mode 100755 index 0000000000..3f400d0b19 --- /dev/null +++ b/src/program/snabbvmx/doc/genbook.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# This shell scripts generates the top-level Markdown structure of the +# Snabb lwAFTR manual. +# +# The authors list is automatically generated from Git history, +# ordered from most to least commits. + +# Script based on src/doc/genbook.sh + +cat < Date: Thu, 27 Oct 2016 09:55:16 +0200 Subject: [PATCH 056/631] Document ingress drop monitor default values --- .../snabbvmx/doc/README.configuration.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/program/snabbvmx/doc/README.configuration.md b/src/program/snabbvmx/doc/README.configuration.md index c2aa6ab12d..c3fbe03c38 100644 --- a/src/program/snabbvmx/doc/README.configuration.md +++ b/src/program/snabbvmx/doc/README.configuration.md @@ -23,7 +23,7 @@ return { cache_refresh_interval = 1, mtu = 1460, }, - setup { + settings = { vlan = 421, } } @@ -31,7 +31,7 @@ return { SnabbVMX defines extra configuration parameters for `ipv4_interface`/`ipv6_interface` (deactivate/activate fragmentation, dynamic/static next-hop resolution, MTU). -In addition, it also allows a `setup` option for extra configuration. +In addition, it also allows a `settings` option for extra configuration. ## Snabb's lwAFTR configuration files @@ -158,12 +158,17 @@ return { Parameters: -- `vlan`: Sets the same VLAN tag for IPv4 and IPv6. If lwAFTR's configuration +* `vlan`: Sets the same VLAN tag for IPv4 and IPv6. If lwAFTR's configuration defines VLAN tags, they take prevalence. -- If `vlan_tag_v4` and `vlan_tag_v6` are defined in lwAFTR configuration, they +* If `vlan_tag_v4` and `vlan_tag_v6` are defined in lwAFTR configuration, they take prevalence. In that case, **SplitV4V6** app is not needed and two virtual interfaces are initialized instead, one for IPv4 and another one for IPv6. Each of them with its own VLAN tag. -- Ingress packet drop parameters initializes several features of the Snabb's +* Ingress packet drop parameters initializes several features of the Snabb's lwAFTR `ingress_drop_monitor` timer. A timer that periodically reports about -the NIC ingress packet drops. +the NIC ingress packet drops. By default, ingress drop monitor is always run. +If not set, it takes the following default values: + * `ingress_drop_monitor`: *flush*. + * `ingress_drop_threshold`: 100000 (packets). + * `ingress_drop_interval`: 1e6 (1 second). + * `ingress_drop_wait`: 20 (seconds). From 625a818904e5f392b007f673e17d448a07e099e3 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 27 Oct 2016 10:36:07 +0200 Subject: [PATCH 057/631] Allow ingress drop monitor deactivation --- src/program/snabbvmx/doc/README.configuration.md | 3 ++- src/program/snabbvmx/lwaftr/lwaftr.lua | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/program/snabbvmx/doc/README.configuration.md b/src/program/snabbvmx/doc/README.configuration.md index c3fbe03c38..41aadf9ba8 100644 --- a/src/program/snabbvmx/doc/README.configuration.md +++ b/src/program/snabbvmx/doc/README.configuration.md @@ -168,7 +168,8 @@ Each of them with its own VLAN tag. lwAFTR `ingress_drop_monitor` timer. A timer that periodically reports about the NIC ingress packet drops. By default, ingress drop monitor is always run. If not set, it takes the following default values: - * `ingress_drop_monitor`: *flush*. + * `ingress_drop_monitor`: *flush*. Other possible values are *warn* for +warning and *off* for deactivating ingress drop monitoring. * `ingress_drop_threshold`: 100000 (packets). * `ingress_drop_interval`: 1e6 (1 second). * `ingress_drop_wait`: 20 (seconds). diff --git a/src/program/snabbvmx/lwaftr/lwaftr.lua b/src/program/snabbvmx/lwaftr/lwaftr.lua index b1b8a85c05..3cdce60cd9 100644 --- a/src/program/snabbvmx/lwaftr/lwaftr.lua +++ b/src/program/snabbvmx/lwaftr/lwaftr.lua @@ -118,6 +118,9 @@ function run(args) if conf.settings then if conf.settings.ingress_drop_monitor then ingress_drop_action = conf.settings.ingress_drop_monitor + if ingress_drop_action == 'off' then + ingress_drop_action = nil + end end if conf.settings.ingress_drop_threshold then ingress_drop_threshold = conf.settings.ingress_drop_threshold From 6650a13ba123d9c3c53a79641122b3e23bd7f44f Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 27 Oct 2016 10:44:00 +0200 Subject: [PATCH 058/631] Add named programs This adds the functionality for programs to claim names and other processes to enumerate processes by their name to retrive their PIDs. These two functions are in core.apps so will be exposed to programs through "engine" --- src/core/app.lua | 77 +++++++++++++++++++ src/core/main.lua | 9 +++ src/program/example_replay/example_replay.lua | 2 + src/program/top/top.lua | 4 +- 4 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/core/app.lua b/src/core/app.lua index 50f05d991a..820dc40d0f 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -12,6 +12,7 @@ local histogram = require('core.histogram') local counter = require("core.counter") local zone = require("jit.zone") local jit = require("jit") +local S = require("syscall") local ffi = require("ffi") local C = ffi.C require("core.packet_h") @@ -25,6 +26,9 @@ local use_restart = false test_skipped_code = 43 +-- Set the directories for the named programs. +named_program_root = shm.root .. "/" .. "by-name" + -- The set of all active apps and links in the system. -- Indexed both by name (in a table) and by number (in an array). app_table, app_array = {}, {} @@ -124,6 +128,65 @@ function configure (new_config) counter.add(configs) end +-- Claims a name for a program so it can be identified by name by other processes. +-- +-- The name given to the function must be unique, if a name has been used before by +-- an active process the function will error displaying an appropriate error message. +-- Any other process can enumerate programs by their name and PID using enumerate_named_programs. +function claim_name(name) + configuration[name] = name + local namedir = "by-name/" .. name + local namedirqualified = named_program_root .. "/" .. name + local piddir = shm.root .. "/" .. S.getpid() + + -- Verify that the by-name directory exists. + shm.mkdir(namedir) + + -- Check if a symlink to a named program with the same name exists, if not, unlink it. + if S.lstat(namedirqualified) then + -- It exists, lets check what the PID is. + local piddir = S.readlink(namedirqualified) + local ps, pe = string.find(piddir, "/[^/]*$") + local pid = tonumber(string.sub(piddir, ps+1, pe)) + if S.kill(pid, 0) then + -- It's a running process, we should display an error message. + error("Name has been claimed by a currently running process: "..namedirqualified) + else + -- It's dead, we should be able to remove the symlink and relink. + S.unlink(namedirqualified) + end + end + + -- Create the new symlink. + local rtn, err = S.symlink(piddir, namedirqualified) + if rtn == -1 then + error("Error creating program name symlink in: "..namedirqualified) + end +end + +-- Enumerates all the named programs with their PID +-- +-- This returns a table programs with the key being the name of the program +-- and the value being the PID of the program. Each program is checked that +-- it's still alive. Any dead program or program without a name is not listed. +function enumerate_named_programs() + local progs = {} + local dirs = S.util.dirtable(named_program_root, true) + if dirs == nil then return progs end + for _, program in pairs(dirs) do + local fq = named_program_root .. "/" .. program + local piddir = S.readlink(fq) + local s, e = string.find(piddir, "/[^/]*$") + local pid = tonumber(string.sub(piddir, s+1, e)) + if S.kill(pid, 0) then + local ps, pe = string.find(fq, "/[^/]*$") + local program_name = string.sub(fq, ps+1, pe) + progs[program_name] = pid + end + end + return progs +end + -- Return the configuration actions needed to migrate from old config to new. -- -- Here is an example return value for a case where two apps must @@ -526,4 +589,18 @@ function selftest () assert(app_table.app3 == orig_app3) -- should be the same main({duration = 4, report = {showapps = true}}) assert(app_table.app3 ~= orig_app3) -- should be restarted + + -- Test claiming and enumerating app names + local basename = "testapp" + claim_name(basename.."1") + claim_name(basename.."2") + + -- Lets check if they can be enumerated. + local progs = enumerate_named_programs() + assert(progs) + assert(progs["testapp1"]) + assert(progs["testapp2"]) + + -- Ensure that trying to take the same name fails + assert(not pcall(claim_name, basename.."1")) end diff --git a/src/core/main.lua b/src/core/main.lua index 611f167152..a0a6d361f7 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -11,6 +11,7 @@ local ffi = require("ffi") local zone = require("jit.zone") local lib = require("core.lib") local shm = require("core.shm") +local app = require("core.app") local C = ffi.C -- Load ljsyscall early to help detect conflicts -- (e.g. FFI type name conflict between Snabb and ljsyscall) @@ -143,6 +144,14 @@ end function shutdown (pid) if not _G.developer_debug and not lib.getenv("SNABB_SHM_KEEP") then shm.unlink("/"..pid) + + -- Look through the named apps and unlink any which are for this process. + local progs = app.enumerate_named_programs() + for name, p in pairs(progs) do + if p == pid then + S.unlink(app.named_program_root .. "/" .. name) + end + end end end diff --git a/src/program/example_replay/example_replay.lua b/src/program/example_replay/example_replay.lua index 1375611109..344ecdf08e 100755 --- a/src/program/example_replay/example_replay.lua +++ b/src/program/example_replay/example_replay.lua @@ -13,6 +13,8 @@ function run (parameters) local pcap_file = parameters[1] local interface = parameters[2] + engine.claim_name("Billybob") + local c = config.new() config.app(c, "capture", pcap.PcapReader, pcap_file) config.app(c, "playback", raw.RawSocket, interface) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 8c1aec02d3..768154daa6 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -32,7 +32,9 @@ function run (args) end function select_snabb_instance (pid) - local instances = shm.children("/") + local instances = {} + -- For named programs there is a "by-name" directory which isn't a process, remove it. + for k,v in pairs(shm.children("/")) do if v ~= "by-name" then instances[k] = v end end if pid then -- Try to use given pid for _, instance in ipairs(instances) do From 0ae21b609722742e45b998dc8834b05e9da0070a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 27 Oct 2016 10:48:23 +0200 Subject: [PATCH 059/631] Remove unintentional addition of claim_name to example_relay I accidently left a claim_name from when I was debugging, this commit removes that. Oops! --- src/program/example_replay/example_replay.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/program/example_replay/example_replay.lua b/src/program/example_replay/example_replay.lua index 344ecdf08e..1375611109 100755 --- a/src/program/example_replay/example_replay.lua +++ b/src/program/example_replay/example_replay.lua @@ -13,8 +13,6 @@ function run (parameters) local pcap_file = parameters[1] local interface = parameters[2] - engine.claim_name("Billybob") - local c = config.new() config.app(c, "capture", pcap.PcapReader, pcap_file) config.app(c, "playback", raw.RawSocket, interface) From c0c64eb018a91c8221ab2f539a484e9b80b0c15e Mon Sep 17 00:00:00 2001 From: Nicola 'tekNico' Larosa Date: Thu, 27 Oct 2016 11:43:38 +0200 Subject: [PATCH 060/631] Changes while reviewing --- .../snabbvmx/doc/README.configuration.md | 36 ++-- src/program/snabbvmx/doc/README.install.md | 32 ++-- src/program/snabbvmx/doc/README.md | 54 +++--- .../snabbvmx/doc/README.troubleshooting.md | 174 +++++++++--------- src/program/snabbvmx/doc/README.userguide.md | 106 +++++------ 5 files changed, 204 insertions(+), 198 deletions(-) diff --git a/src/program/snabbvmx/doc/README.configuration.md b/src/program/snabbvmx/doc/README.configuration.md index 41aadf9ba8..111de0df0a 100644 --- a/src/program/snabbvmx/doc/README.configuration.md +++ b/src/program/snabbvmx/doc/README.configuration.md @@ -2,12 +2,12 @@ ## SnabbVMX's configuration file -SnabbVMX has its own configuration file (Example: **snabbvmx-lwaftr-xe0.cfg**). -Configuration file is actually a Lua file, that is processed by `snabbvmx lwaftr` -command. +SnabbVMX has its own configuration file (example: **snabbvmx-lwaftr-xe0.cfg**). +The configuration file is actually a Lua file, that is processed by the +`snabbvmx lwaftr` command. It contains a reference to a Snabb's **lwAFTR** configuration file (which -contains a reference to a binding-table). +contains a reference to a binding table). **snabbvmx-lwaftr-xe0.cfg** @@ -35,9 +35,9 @@ In addition, it also allows a `settings` option for extra configuration. ## Snabb's lwAFTR configuration files -SnabbVMX's configuration file `lwaftr` attribute points out to a Snabb's lwAFTR +SnabbVMX's configuration file `lwaftr` attribute points to a Snabb's lwAFTR configuration file. This configuration file keeps a reference to a -binding-table, among other information, which is an important piece of a +binding table, among other information, which is an important piece of a lwAFTR deployment. Here is how a lwAFTR configuration file looks like: @@ -61,7 +61,7 @@ policy_icmpv4_incoming = DROP, policy_icmpv4_outgoing = DROP, ``` -And here is its referred binding-table: +And here is its referred binding table: ``` psid_map { @@ -83,11 +83,11 @@ softwires { Some of the lwAFTR's configuration fields are of special relevance for SnabbVMX. Although SnabbVMX can specify its own MTU and VLAN values, if those -attributes are defined in a lwAFTR configuration file they always take -precedence. +attributes are also defined in a lwAFTR configuration file, the latter always +take precedence. Please refer to Snabb's lwAFTR documentation for a detailed description about -lwAFTR's configuration file and binding-table (Chapters 3 and 4). +lwAFTR's configuration file and binding table (Chapters 3 and 4). ## Configuration examples @@ -138,7 +138,7 @@ Parameters: - `next_hop_mac`: MAC address of the nexthop. Outgoing IPv4 or IPv6 packets will use this MAC address as Ethernet destination address. -- `fragmentation`: Boolean field. Indicated whether IPv4 or IPv6 packets get +- `fragmentation`: Boolean field. Selects whether IPv4 or IPv6 packets get fragmented by the lwAFTR in case packets are too big (larger than MTU size). ### Additional setup @@ -159,14 +159,14 @@ return { Parameters: * `vlan`: Sets the same VLAN tag for IPv4 and IPv6. If lwAFTR's configuration -defines VLAN tags, they take prevalence. +defines VLAN tags, they take precedence. * If `vlan_tag_v4` and `vlan_tag_v6` are defined in lwAFTR configuration, they -take prevalence. In that case, **SplitV4V6** app is not needed and two virtual -interfaces are initialized instead, one for IPv4 and another one for IPv6. -Each of them with its own VLAN tag. -* Ingress packet drop parameters initializes several features of the Snabb's -lwAFTR `ingress_drop_monitor` timer. A timer that periodically reports about -the NIC ingress packet drops. By default, ingress drop monitor is always run. +take precedence. In that case, **SplitV4V6** app is not needed and two virtual +interfaces are initialized instead, one for IPv4 and another one for IPv6, +each of them with its own VLAN tag. +* Ingress packet drop parameters initialize several features of the Snabb's +lwAFTR `ingress_drop_monitor` timer. Periodically reports about the NIC +ingress packet drops. By default, ingress drop monitor is always run. If not set, it takes the following default values: * `ingress_drop_monitor`: *flush*. Other possible values are *warn* for warning and *off* for deactivating ingress drop monitoring. diff --git a/src/program/snabbvmx/doc/README.install.md b/src/program/snabbvmx/doc/README.install.md index 7f36eae2e6..ceffdb29df 100644 --- a/src/program/snabbvmx/doc/README.install.md +++ b/src/program/snabbvmx/doc/README.install.md @@ -1,9 +1,9 @@ -# Instalation +# Installation ## How to install it -Snabb is a programming network toolkit as well as ready-to-use network -utilities suite. Every Snabb executable features several Snabb programs: +Snabb is a programming network toolkit as well as a ready-to-use network +utilities suite. The Snabb executable features several Snabb programs: ```bash $ sudo ./snabb @@ -45,7 +45,8 @@ Example: There is no specific script to install a snabb executable. Once it's built, a snabb executable includes all its dependencies, including a LuaJIT interpreter, in a single binary. Thus, it's possible to relocate a snabb executable to any -folder in the system. Move it to folder in PATH, so the system can alway locate it: +folder in the system. Move it to a folder in PATH, so the system can always +locate it: ```bash $ echo $PATH @@ -55,9 +56,9 @@ $ echo $PATH $ cp snabb /opt/local/bin ``` -When snabb executable is renamed to one of its featured programs, it will always -the program is named after. For instance, to always run snabbvmx simply rename -snabb to snabbvmx. +When the snabb executable is renamed to one of its featured programs, it will +run the program it's named after. For instance, to always run snabbvmx simply +rename snabb to snabbvmx. ```bash $ mv snabb snabbvmx @@ -71,14 +72,15 @@ Usage: ## SnabbVMX tools -SnabbVMX program (**program/snabbvmx/**) features a series of subcommands or tools: +The SnabbVMX program (**program/snabbvmx/**) features a series of subcommands +or tools: -- **check**: Used to verify the correctness of the lwAFTR logic. -- **nexthop**: Used to retrieve the nexthop cached values (available in shared memory). -- **query**: Used to check out the counter values of a running SnabbVMX instance. -- **lwaftr**: Main program. Set ups SnabbVMX network design and runs it. -- **top**: Similar to Snabb's top. Prints out Gb and Mpps in IPv4 and IPv6 interfaces. -Includes reports about counters and ingress-packet-drops. +- **check**: Verifies the correctness of the lwAFTR logic. +- **nexthop**: Retrieves the nexthop cached values (available in shared memory). +- **query**: Checks out the counter values of a running SnabbVMX instance. +- **lwaftr**: Main program. Sets up the SnabbVMX network design and runs it. +- **top**: Similar to Snabb's top. Prints out Gb and Mpps in IPv4 and IPv6 + interfaces. Includes reports about counters and ingress-packet-drops. There is an additional program in snabb called **packetblaster**. Packetblaster includes a *lwaftr* mode. This mode is very useful to generate live traffic matching @@ -93,4 +95,4 @@ a binding-table. ``` Please check the [How to use it?](README.userguide.md) chapter for a more -detailed view on each tool. +detailed view of each tool. diff --git a/src/program/snabbvmx/doc/README.md b/src/program/snabbvmx/doc/README.md index ca3f017f2a..4410406007 100644 --- a/src/program/snabbvmx/doc/README.md +++ b/src/program/snabbvmx/doc/README.md @@ -7,12 +7,12 @@ by Snabb. For a detailed explanation of what's an lwAFTR, please refer to lwAFTR docs. Snabb's lwAFTR app, in addition to managing lw4o6 traffic (IPv4 and IPv6 packets), -it also manages non-lwAFTR traffic such as pings to lwAFTR interfaces, ARP and -NDP resolution. This traffic is managed by addition applications connected to +also manages non-lwAFTR traffic such as pings to lwAFTR interfaces, ARP and +NDP resolution. This traffic is managed by additional applications connected to the lwAFTR network design in program/lwaftr/run.lua. Management of this type -of traffic is a requiremennt in RFC 7596. +of traffic is a requirement in RFC 7596. -SnabbVMX uses a different approach. It setups a network design where only lw4o6 +SnabbVMX uses a different approach. It sets up a network design where only lw4o6 packets are managed by the lwAFTR app, while everything else is forwarded to a VM which is in charge of providing ping, ARP and NDP resolution. @@ -22,7 +22,7 @@ SnabbVMX's app network: ![SnabbVMX](.images/snabbvmx.png) -SnabbVMX works in one single 10Gb NIC. Incoming traffic gets to an Intel82599 app +SnabbVMX works on a single 10Gb NIC. Incoming traffic gets to an Intel82599 app managed by Snabb. App V4V6 splits traffic between IPv4 or IPv6 and forwards it to the correspondent @@ -32,8 +32,8 @@ to the lwAFTR app or to the VM, based on unicast/broadcast destination MAC, matching local IPv4 destination or IPv4-in-IPv6. App VhostUser communicates with a VM run by QEMU using the VhostUser network -backend available in QEMU. Communication between both processes happens via a -socket. +backend available in QEMU. Communication between the two processes happens via +a socket. App V4V6, in combination with a Tap app, allows monitoring of packets coming in and out of the physical port based on a matching IPv4 address, either as @@ -71,20 +71,20 @@ return { lwaftr points to Snabb's lwAFTR configuration file. -Othe attributes are further refined by SnabbVMX. Attributes `ipv6_interface` +Other attributes are further refined by SnabbVMX. Attributes `ipv6_interface` and `ipv4_interface` are mandatory and ipv4_interface must include the IP addresses of the correspondent lwAFTR IPv4 interfaces. The IPv4 address is used to send matching packets to the VMX instead of trying to find a match in the binding table for encap. Attribute `settings` may include a `vlan` tag attribute, which can be either -false in case VLAN tagging is not enabled or a VLAN tag number (0-4095). +false, in case VLAN tagging is not enabled, or a VLAN tag number (0-4095). Reassembly and fragmentation are deactivated by default in SnabbVMX. In case they are needed, set `ipv6_interface.fragmentation = true`. -Interfaces also allow setting of IPv4 and IPv6 ingress/egree filters. The -filters are expessed as a packet-filtering expression (like tcpdump) or point +Interfaces also allow setting of IPv4 and IPv6 ingress/egress filters. The +filters are expressed as a packet-filtering expression (like tcpdump) or point to a file containing a pf expression. #### Components overview @@ -93,20 +93,20 @@ List of apps used in SnabbVMX network design: * **V4V6** (`apps/lwaftr/V4V6.lua`): Categorizes traffic as IPv4 and IPv6 and forwards it to the correspondent link. -* **Tap** (`apps/tap/tap.lua`): Used to monitor traffic in V4V6 via a Tap -interface. Matching IPv4 address is controlled by "lwaftr snabbvmx monitor". -* **ReassemblerV6**/**FragmenterV6** (`apps/lwaftr/ipv6_apps.lua`): IPv6 reassembler and fragmenter apps. -* **ReassemblerV4**/**FragmenterV4** (`apps/lwaftr/ipv4_apps.lua`): IPv6 reassembler and -fragmenter apps. -* **Next-hop Forwarder** (`apps/nh_fwd/nh_fwd.lua`): It contains to forwarding apps: -nh_fwd6 and nh_fwd4. The apps forwards incoming traffic to a service, in this -case to a lwAFTR app, and to a VM. Routing decisions are based on next-hop -MAC address learned from the VM and by IP destination address of incoming +* **Tap** (`apps/tap/tap.lua`): Monitors traffic in V4V6 via a Tap interface. +The matching IPv4 address is controlled by `lwaftr snabbvmx monitor`. +* **ReassemblerV6**/**FragmenterV6** (`apps/lwaftr/ipv6_apps.lua`): IPv6 +reassembler and fragmenter apps. +* **ReassemblerV4**/**FragmenterV4** (`apps/lwaftr/ipv4_apps.lua`): IPv4 +reassembler and fragmenter apps. +* **Next-hop Forwarder** (`apps/nh_fwd/nh_fwd.lua`): Contains two forwarding apps: +`nh_fwd6` and `nh_fwd4`. The apps forward incoming traffic to a service, in this +case to a lwAFTR app, and to a VM. Routing decisions are based on the next-hop +MAC address learned from the VM and by the IP destination address of incoming packets. -* **lwAFTR** (`apps/lwaftr/lwaftr.lua`): Implements a lwAFTR component as specified -in RFC7569. -* **VhostUser** (`apps/vhost/vhost_user.lua`): A driver complying Virtio which allows -a program in user space, in this case Snabb, to act as an hypervisor of the -networking side of a guest. Similarly, the guest side has to be managed by a -VhostUser network backend (already built-in in QEMU). - +* **lwAFTR** (`apps/lwaftr/lwaftr.lua`): Implements a lwAFTR component as +specified in RFC7569. +* **VhostUser** (`apps/vhost/vhost_user.lua`): A driver complying with Virtio +which allows a program in user space, in this case Snabb, to act as an hypervisor +of the networking side of a guest. Similarly, the guest side has to be managed +by a VhostUser network backend (already built-in in QEMU). diff --git a/src/program/snabbvmx/doc/README.troubleshooting.md b/src/program/snabbvmx/doc/README.troubleshooting.md index 34e9e41158..c6e177049d 100644 --- a/src/program/snabbvmx/doc/README.troubleshooting.md +++ b/src/program/snabbvmx/doc/README.troubleshooting.md @@ -2,31 +2,31 @@ ## Troubleshooting -### Troubleshooting in the lwAFTR +### Troubleshooting lwAFTR If traffic managed by the lwAFTR component is not responding as expected: packets that should get decapsulate don't match a softwire, some packets are dropped, -traffic don't get in, etc, the most convenient is to inspect lwAFTR counters -to diagnose the problem. Snabb's `lwaftr query` tool can be used to obtain +traffic don't get in, etc, it's convenient to inspect lwAFTR counters to +diagnose the problem. The Snabb's `lwaftr query` tool can be used to obtain the counter values of a lwAFTR instance. -Snabb's lwAFTR manual includes a chapter covering troubleshooting and counters, +The Snabb's lwAFTR manual includes a chapter covering troubleshooting and counters, with charts about the most common lwAFTR paths (encapsulation, decapsulation, -hairpinning, etc). Please refer to that guide for troubleshooting in the -lwAFTR logic (Chapter 8 - Counters). +hairpinning, etc). Please refer to that guide for troubleshooting the +lwAFTR logic (chapter 8 - Counters). The manual also includes a section covering troubleshooting related with running a lwAFTR instance. This information can be useful in case of running the lwAFTR as a standalone application and not via SnabbVMX. -### Troubleshooting in SnabbVMX +### Troubleshooting SnabbVMX -This section covers most common problems when running a SnabbVMX instance. +This section covers the most common problems when running a SnabbVMX instance. -#### Cannot start SnabbVMX instance +#### Cannot start the SnabbVMX instance -`Description`: When running a snabbvmx instance an error reporting `failed to lock -`. ```bash $ sudo ./snabb snabbvmx lwaftr --conf snabbvmx-lwaftr.cfg --id xe1 \ @@ -45,25 +45,25 @@ Snabb instance running on the same NIC (in this example, `0000:81:00.0`). #### SnabbVMX running but not receiving traffic -`Description`: `snabbvmx lwaftr` instance is running on a NIC but `snabbvmx top` -reports the SnabbVMX instance is receiving no traffic. +`Description`: a `snabbvmx lwaftr` instance is running on a NIC, but `snabbvmx +top` reports that the SnabbVMX instance is receiving no traffic. `Solution`: This is a common problem which might be originated by various causes. 1. If no traffic is received at all, the most likely cause is that the selected -lwAFTR binding table doesn't contain any valid softwire for incoming traffic. +lwAFTR binding table contains no valid softwires for incoming traffic. 2. Another possible cause is that SnabbVMX is running with dynamic nexthop resolution, but dynamic nexthop resolution is not working. #### Dynamic nexthop resolution is not working -`Description`: SnabbVMX is running with dynamic nexthop resolution but no traffic +`Description`: SnabbVMX is running with dynamic nexthop resolution, but no traffic is leaving the lwAFTR. -`Solution`: Check SnabbVMX is indeed running with dynamic nexthop resolution. -SnabbVMX's configuration file should have attribute `cache_refresh_interval` set to a -value higher than 0. +`Solution`: Check that SnabbVMX is indeed running with dynamic nexthop resolution. +SnabbVMX's configuration file should have a `cache_refresh_interval` attribute +set to a value higher than 0. ```bash ipv6_interface = { @@ -75,7 +75,7 @@ ipv4_interface = { }, ``` -If that's correct, the likely caused is that refreshment packets are not arriving +If that's correct, the likely cause is that refreshment packets are not arriving to the VM. A refreshment packet is a packet that is sent periodically to the VM to trigger nexthop cache resolution. @@ -88,24 +88,24 @@ sudo ./snabb snabbvmx lwaftr --id xe0 --conf snabbvmx-lwaftr-xe0.cfg ``` If that's correct, another cause is that the selected MAC address (02:aa:aa: -aa:aa:aa, in the example above), doesn't match the lwAFTR's configuration -attributes `aftr_mac_inet_side` and `aftr_mac_b4_side`. Check next section at -the bottom for more details about that. +aa:aa:aa in the example above) doesn't match the lwAFTR's configuration +attributes `aftr_mac_inet_side` and `aftr_mac_b4_side`. Check the bottom of +the next section for more details. #### SnabbVMX doesn't respond to IPv4 or IPV6 pings `Description`: Trying to ping SnabbVMX on one of its IPv4 or IPv6 interfaces gets no response. -`Solution`: Double-check the target address is indeed lwAFTR IPV4 or IPv6 -addresses. Steps: +`Solution`: Check that the target address is indeed the lwAFTR IPV4 and IPv6 +address. Steps: 1. Open the selected SnabbVMX configuration file and go to its lwAFTR configuration. -2. Check the values of attributes `aftr_ipv6_ip` and `aftr_ipv4_ip` are +2. Check that the values of attributes `aftr_ipv6_ip` and `aftr_ipv4_ip` are equals to the target address. -In case the address is correct, double-check SnabbVMX MAC address is the same +In case the address is correct, check that the SnabbVMX MAC address is the same as `aftr_mac_inet_side` and `aftr_mac_b4_side`. In SnabbVMX these values must be the same, as there's a single NIC for both interfaces. @@ -121,16 +121,18 @@ root 16115 0.0 0.0 133856 2936 pts/2 S+ 22:39 0:00 #### SnabbVMX decapsulation path works but not encapsulation -`Description`: `snabbvmx top` reports running SnabbVMX instance is able to -decapsulate packets but not to encapsulate. +`Description`: `snabbvmx top` reports that the running SnabbVMX instance is able +to decapsulate packets, but not to encapsulate them. +``` lwaftr (rx/tx/txdrop in Mpps) rx tx rxGb txGb txdrop lwaftr_v6 1.53 0 4.80 0 0.000022 lwaftr_v4 0 1.53 0 4.31 0 +``` -`Solution`: If the decapsulation path works that means lwAFTR is able to -decapsulate IPv6 packets coming from the B4. That means the IPv4 source address -and source port of the IPv6 encapsulated packet matches a softwire in the +`Solution`: If the decapsulation path works, then the lwAFTR is able to +decapsulate IPv6 packets coming from the B4: therefore the IPv4 source address +and source port of the IPv6 encapsulated packet match a softwire in the binding table. However, when a packet arrives to the lwAFTR from the Internet the packet is dropped, thus RX is 0 in lwaftr_v4. @@ -138,12 +140,12 @@ The destination address and port of an incoming IPv4 packet should match a softwire in the binding table. What's the destination address and port of incoming packets? Is the lwAFTR using a VLAN tag for its IPv4 interface? Most likely incoming packets are not VLAN tagged. Check if SnabbVMX's config file -has a VLAN tag set. Check also if the referred lwAFTR config file has a VLAN +has a VLAN tag set. Also check if the referred lwAFTR config file has a VLAN tag set. What's the MTU size of the IPv4 interface? Check that value in SnabbVMX and lwAFTR configuration file. If the MTU size is too small, packets will be -fragmented. A extremely small MTU size and fragmentation disable will cause +fragmented. A extremely small MTU size and disabled fragmentation will cause most of the packets to be dropped. #### Packets don't get mirrored @@ -153,35 +155,35 @@ nothing gets in. `Solution`: -1. Check you're running SnabbVMX with mirroring enabled. -2. Check you're running `lwaftr monitor` pointing to the IPv4 address you would -like to monitor. For testing purposes, set `lwaftr monitor` to `all`. -3. Check your tap interface is up. +1. Check that you're running SnabbVMX with mirroring enabled. +2. Check that you're running `lwaftr monitor` pointing to the IPv4 address you +would like to monitor. For testing purposes, set `lwaftr monitor` to `all`. +3. Check that your tap interface is up. --- ## Tests overview -Test are useful to detect bugs in the `snabbvmx` and double-check everything -is working as expected. +Tests are useful to detect bugs in the `snabbvmx` and double-check that +everything is working as expected. SnabbVMX features three types of tests: * Lua selftests: Unit tests for several modules (**apps/lwaftr/V4V6.lua** and **apps/lwaftr/nh_fwd.lua**). -* Bash selftest: Complex setups to test certain functionality (**program/ +* Bash selftests: Complex setups to test certain functionality (**program/ snabbvmx/tests/selftest.sh** and **program/snabbvmx/tests/nexthop/ selftest.sh**). -* End-to-end tests: Particular end-to-end test cases for SnabbVMX +* End-to-end tests: Specific end-to-end test cases for SnabbVMX (**program/snabbvmx/tests/end-to-end/selftest.sh**). -Usually Lua selftests and Bash selftest won't fail as these test must successfully -pass on every Snabb deliverable. End-to-end tests won't fail either, but it is -interesting to learn how to add new SnabbVMX's end-to-end tests to diagnose -potential bugs. +Usually Lua selftests and Bash selftests won't fail, as these tests must +successfully pass on every Snabb deliverable. End-to-end tests won't fail +either, but it's interesting to learn how to add new SnabbVMX's end-to-end +tests to diagnose potential bugs. All these tests are run by Snabb's Continuous Integration subsystem (snabb-bot). -The test can also be run when executing **make test** (only if Snabb's source +The tests can also be run when executing **make test** (only if the Snabb source is available). ```bash @@ -202,13 +204,13 @@ TEST program/lwaftr/tests/soaktest/selftest.sh Execution of a test can return 3 values: * **TEST**: The test run successfully (exit code 0) -* **SKIPPED**: The test was skipped, usually because it needs to access to -a physical NIC and its PCI address was not setup (exit code 43). -* **ERROR**: Test test finished unexpectedly (exit code > 0 and != 43). +* **SKIPPED**: The test was skipped, usually because it needs access to +a physical NIC and its PCI address was not set up (exit code 43). +* **ERROR**: The test finished unexpectedly (exit code > 0 and != 43). ### Lua selftests -* **apps/lwaftr/V4V6.lua**: Builds customized IPv4 and IPv4-in-IPv6 packets +* **apps/lwaftr/V4V6.lua**: Builds customized IPv4 and IPv4-in-IPv6 packets, and joins the packets to a single link (*test_join*) or splits the packets to two different links (*test_split*). * **apps/lwaftr/nh_fwd.lua**: Builds customized packets and test the 3 code @@ -253,23 +255,23 @@ NDP request to lwAFTR (Good VLAN): OK NDP request to lwAFTR (Bad VLAN): OK ``` -NOTE: To successfully run the test `SNABB_PCI0` and `SNABB_PCI1` cards must -be wired together. +NOTE: To successfully run the test, the `SNABB_PCI0` and `SNABB_PCI1` cards must +be wired to each other. The test goes through several steps: 1. Run **SnabbVMX** on NIC **SNABB_PCI0**. 2. Run **QEMU**. 3. Configure VM eth0 interface with MAC, IPv4/IPv6 address, ARP & NDP cache table. -4. Runs **tcpreplay** on NIC SNABB_PCI1. Sample packets reach the VM. +4. Run **tcpreplay** on NIC SNABB_PCI1. Sample packets reach the VM. 5. Outgoing packets from the VM are mirrored to a tap interface (tap0). -6. Captures **responses on tap0** and compares them with expected results. +6. Capture **responses on tap0** and compare them with the expected results. -The input data as well as the expected output is at `program/snabbvmx/tests/pcap`. +The input data, as well as the expected output, is at `program/snabbvmx/tests/pcap`. -The test validated VLAN packets too. However, there are not VLAN tagged versions +The test validates VLAN packets too. However, there are no VLAN tagged versions of the expected output. The reason is that it is the NIC which tags and untags -a packet. Since the packet have not leave the NIC yet, they come out from the VM +a packet. Since the packet did not leave the NIC yet, they come out from the VM untagged. ```bash @@ -279,7 +281,7 @@ reading from file arp-request-to-lwAFTR.pcap, link-type EN10MB (Ethernet) ethertype ARP, Request who-has 10.0.1.1 tell 10.0.1.100, length 28 ``` -The other bash selftest, validates correct resolution of the nexthop by the VM. +The other bash selftest validates correct resolution of the nexthop by the VM. ```bash $ sudo SNABB_PCI0=03:00.0 SNABB_PCI1=83:00.0 \ @@ -293,17 +295,17 @@ The test goes through several steps: 1. Run SnabbVMX on SNABB_PCI0. 2. Run VM. -3. Send packets in loop to SNABB_PCI1 during 10 seconds. Packets reach the VM. +3. Send packets in loop to SNABB_PCI1 for 10 seconds. Packets reach the VM. ```bash $ packetblaster replay -D 10 $PCAP_INPUT/v4v6-256.pcap $SNABB_PCI1 ``` -4. Retrieve out nexthop values (snabbvmx nexthop). +4. Retrieve nexthop values (snabbvmx nexthop). 5. Compare to expected values. 6. Timeout if 10 seconds elapsed. -NOTE: Currently the test is not working correctly. The returned MAC should be +NOTE: Currently the test is not working correctly: the returned MAC should be `02:99:99:99:99:99`. ### End-to-end tests @@ -311,18 +313,18 @@ NOTE: Currently the test is not working correctly. The returned MAC should be **program/snabbvmx/tests/end-to-end/selftest.sh**: Runs end-to-end tests (normal and VLAN). -The end-to-end tests is a test suite that tests the correctness of the lwAFTR +The end-to-end tests are a test suite that tests the correctness of the lwAFTR logic. Files in **program/snabbvmx/tests/end-to-end/**: * **test_env.sh**: Contains the test cases. -* **core-end-to-end.sh**: Actually runs the test cases using **snabbvmx check**. +* **core-end-to-end.sh**: Runs the test cases using **snabbvmx check**. * **data/**: Directory containing sample pcap input, expected pcap output, configuration files and binding tables. -* **end-to-end.sh**: Runs core-end-to-end.sh on normal packets. -* **end-to-end-vlan.sh**: Runs core-end-to-end.sh on VLAN packets. -* **selftest.sh**: Runs both end-to-end.sh and end-to-end-vlan.sh +* **end-to-end.sh**: Runs **core-end-to-end.sh** on normal packets. +* **end-to-end-vlan.sh**: Runs **core-end-to-end.sh** on VLAN packets. +* **selftest.sh**: Runs both **end-to-end.sh** and **end-to-end-vlan.sh** To run SnabbVMX's end-to-end test suite: @@ -334,7 +336,7 @@ Test passed All end-to-end lwAFTR tests passed. ``` -The end-to-end test suite relies on `snabbvmx check` to run each test. +The end-to-end test suite relies on `snabbvmx check` to run each test: ```bash @@ -346,38 +348,38 @@ Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP Parameters: - **CONF**: SnabbVMX configuration file. -- **V4-IN.PCAP**: Incoming IPv4 packets (from inet). +- **V4-IN.PCAP**: Incoming IPv4 packets (from Internet). - **V6-IN.PCAP**: Incoming IPv6 packets (from b4). -- **V4-OUT.PCAP**: Outgoing IPv4 packets (to inet, decapsulate). +- **V4-OUT.PCAP**: Outgoing IPv4 packets (to Internet, decapsulated). - **V6-OUT.PCAP**: Outgoing IPv6 packets (to b4, encapsulated) - **[COUNTERS.LUA]**: Lua file with counter values. -Although SnabbVMX works in one single interface, snabbvmx check requires that -the packet split is already done and provides an split output too. +Although SnabbVMX works on a single interface, `snabbvmx check` requires that +the packet split is already done and provides a split output too. Snabb's lwAFTR includes an end-to-end test suite counterpart. In most cases, -the lwAFTR's correctness will be test via Snabb's lwAFTR end-to-end tests. +the lwAFTR's correctness will be tested via Snabb's lwAFTR end-to-end tests. However, since SnabbVMX uses a different configuration file, the network design that it brings up might be slightly different than Snabb's lwAFTR. For instance, -Snabb's lwAFTR fragmentation is always activated while in SnabbVMX is an optional +Snabb's lwAFTR fragmentation is always active, while in SnabbVMX it is an optional argument, either for IPV4 and IPv6 interfaces. -Modifications in the app chain might run executed lwAFTR's data plane in a different -way, bringing up conditions that were not covered by lwAFTR's data-plane. For -this reason, a specific end-to-end test suite was added to SnabbVMX. A brand-new -test set that covers specifics needs of SnabbVMX. If a bug is found most likely -its resolution will happen in Snabb's lwAFTR code, resulting into the addition of -a new test in lwAFTR's test-suite. +Modifications in the app chain might execute the lwAFTR's data plane in a different +way, bringing up conditions that were not covered by the lwAFTR's data-plane. +For this reason a specific end-to-end test suite was added, covering specifics +needs of SnabbVMX. If a bug is found, its resolution will most likely happen +in Snabb's lwAFTR code, resulting in the addition of a new test to the lwAFTR's +test suite. ## How to write a SnabbVMX end-to-end test -Imagine you have detected an error in the lwAFTR. The first step is to obtain -the configuration file that SnabbVMX was using, as well as a copy of lwAFTR's +If you detected an error in the lwAFTR, the first step is to obtain the +configuration file that SnabbVMX was using, as well as a copy of lwAFTR's configuration and binding table. With that information and knowing the error report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get - decapsulate, etc), you craft a hand-made packet that meets the testing case. +decapsulated, etc), you craft a hand-made packet that meets the testing case. -Now we can check what the lwAFTR would produce: +Now we can check what the lwAFTR produces: ```bash sudo ./snabb snabbvmx -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap empty.pcap \ @@ -386,7 +388,7 @@ sudo ./snabb snabbvmx -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap empty.pcap \ The flag `-r` generates a counters file. -Check your output matches what you expected: +Check that your output matches what you expect: ```bash $ tcpdump -qns 0 -ter empty.pcap @@ -396,5 +398,5 @@ reading from file empty.pcap, link-type EN10MB (Ethernet) Checking what values are in the counters can give you a hint about whether things are working correctly or not. -Tip: Packets always arrive only in one interface, but the output might be -empty for both interfaces, non-empty and empty or non-empty for both cases. +Tip: packets always arrive only in one interface, but the output might be +empty for both interfaces, non-empty, and empty or non-empty for both cases. diff --git a/src/program/snabbvmx/doc/README.userguide.md b/src/program/snabbvmx/doc/README.userguide.md index c0c5cc2a33..c2eb28bebc 100644 --- a/src/program/snabbvmx/doc/README.userguide.md +++ b/src/program/snabbvmx/doc/README.userguide.md @@ -1,13 +1,13 @@ # User guide -The following chapter teaches you how to get SnabbVMX up-and-running. This -chapter includes references to concepts and terms not covered yet. Please -refer to other sections in this manual in case of doubt. +This guide teaches you how to get SnabbVMX up-and-running, including references +to concepts and terms not covered yet. Please refer to other sections in this +manual in case of doubt. ## SnabbVMX lwaftr -`snabbvmx lwaftr` is the main program. It setups the app network design and pass - it to Snabb's engine to run it. +`snabbvmx lwaftr` is the main program. It sets up the app network design, and +passes it to the Snabb's engine to run it. ```bash $ sudo ./snabb snabbvmx lwaftr @@ -26,7 +26,7 @@ Arguments: --sock Socket path for virtio-user interfaces ``` -The example below use static next-hop resolution so a VM is not needed. This +The example below uses static next-hop resolution, so a VM is not needed. This setup is useful to test a lwAFTR data-plane. **snabbvmx-lwaftr-xe0.cfg** @@ -110,8 +110,8 @@ Ingress drop monitor: flush (threshold: 100000 packets; ``` Now we should send valid lwAFTR traffic to SnabbVMX. One way of doing it is -using a Snabb tool called **packetblaster**. Packetblaster has a lwaftr mode -that generates valid lwAFTR packets for a given binding-table configuration. +using a Snabb tool called **packetblaster**. Packetblaster has a *lwaftr* mode +that generates valid lwAFTR packets for a given binding table configuration. ```bash $ sudo ./snabb packetblaster lwaftr --src_mac 02:02:02:02:02:02 \ @@ -122,8 +122,8 @@ $ sudo ./snabb packetblaster lwaftr --src_mac 02:02:02:02:02:02 \ ## SnabbVMX top -`snabbvmx top` prints out information of a running SnabbVMX instance. There might -be more than one Snabb process running: +`snabbvmx top` prints out information about a running SnabbVMX instance. There +might be more than one Snabb process running: ```bash $ sudo ./snabb top @@ -132,7 +132,7 @@ Multiple Snabb instances found. Select one: 11958 ``` -Which one is my SnabbVMX process? Luckily, SnabbVMX top allows to query per 'id': +Which one is my SnabbVMX process? Luckily, SnabbVMX top allows to query by "id": ```bash $ sudo ./snabb snabbvmx top xe0 @@ -154,8 +154,8 @@ lwaftr_v6 out-ipv6-frag-not 0 0 nic ifInDiscards 1,547,333 47,449 ``` -1.53 MPPS in both interfaces rx/tx. We are running at line-rate, although there - are packet drops. +1.53 MPPS in both rx/tx interfaces. We are running at line-rate, although there +are packet drops. ## SnabbVMX query @@ -205,14 +205,14 @@ SnabbVMX query prints out all the counters of a SnabbVMX instance in XML format: ``` -Useful for interacting with other systems in the network. +It's useful for interacting with other systems in the network. -Snabb's lwAFTR features a query tool too. This tool is aimed to query lwAFTR's -counters. Snabb's `lwaftr query` is covered in the last section of this chapter. +Snabb's lwAFTR also features a tool that queries lwAFTR's counters: Snabb's +`lwaftr query` is covered in the last section of this chapter. ## SnabbVMX check -`snabbvmx check` is an utility tool use to validate lwAFTR correctnes. +`snabbvmx check` is a utility that validates lwAFTR correctness. SnabbVMX has its own version of it. ```bash @@ -221,52 +221,53 @@ Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP [COUNTERS.LUA] ``` -Using check is the step prior to add a new test in the end-to-end test suite. +Using `check` is the step prior to adding a new test to the end-to-end test suite. -**CONF** is a SnabbVMX configuration file and **V4-IN** and **V6-IN** are the -incoming V4 and V6 interfaces. The tools reads packets from a pcap file. **V4-OUT** +**CONF** is a SnabbVMX configuration file, and **V4-IN** and **V6-IN** are the +incoming V4 and V6 interfaces. The tool reads packets from a pcap file. **V4-OUT** and **V6-OUT** are the resulting packets after the lwAFTR processing. How the packets are processed depends on the configuration file and the binding table. -Although SnabbVMX works in one single interface, snabbvmx check requires that -the packet split is already done and provides an split output too. +Although SnabbVMX works on a single interface, `snabbvmx check` requires that +the packet split is already done and provides a split output too. -Imagine you have detected an error in the lwAFTR. First step is to obtain the -configuration that SnabbVMX was using, as well as a copy of lwAFTR's -configuration and binding table. With that information and knowing the error +If you detected an error in the lwAFTR, the first step is to obtain the +configuration file that SnabbVMX was using, as well as a copy of lwAFTR's +configuration and binding table. With that information and knowing the error report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get -decapsulate), you hand-made a packet that meets the testing case. Now we can -check what the lwAFTR will produce: +decapsulated, etc), you craft a hand-made packet that meets the testing case. + +Now we can check what the lwAFTR produces: ```bash sudo ./snabb snabbvmx -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap empty.pcap \ /tmp/outv4.pcap /tmp/outv6.pcap counters.lua ``` -The flag -r generates a counters file. +The flag `-r` generates a counters file. -Check your output matches what you expected: +Check that your output matches what you expect: ```bash $ tcpdump -qns 0 -ter empty.pcap reading from file empty.pcap, link-type EN10MB (Ethernet) ``` -Checking what values are in counters can tell you if things are working -correctly or not. +Checking what values are in the counters can give you a hint about whether +things are working correctly or not. -Tip: Packets always arrive only in one interface, but the output might be -empty for both interfaces, non-empty and empty or non-empty for both cases. +Tip: packets always arrive only in one interface, but the output might be +empty for both interfaces, non-empty, and empty or non-empty for both cases. ## Other related tools ### Snabb's lwaftr query Snabb's `lwaftr query` command can be used to print out counter's values of -a running lwAFTR instance (that also includes a SnabbVMX instance). In case of -query a SnabbVMX instance, the instance can be referred by its `id`. +a running lwAFTR instance (that also includes a SnabbVMX instance). When +querying a SnabbVMX instance, the instance can be referred to by its `id`. -Counters are useful to debug and understand what data-paths are being taken. +Counters are useful to debug and understand what data paths are being taken. Running `snabb lwaftr query ` lists all non-zero counters of a lwAFTR's instance: @@ -296,8 +297,8 @@ out-ipv6-bytes: 509,917,643,140 out-ipv6-packets: 1,297,268,099 ``` -It is possible to pass a filter expression to query only matching counters. -The example below prints out only outgoing related counters. +It is possible to pass a filter expression to query only the matching counters. +The example below prints out only the outgoing related counters. ```bash $ sudo ./snabb lwaftr query 11958 out @@ -314,8 +315,8 @@ out-ipv6-packets: 1,583,770,321 ### Snabb's lwaftr monitor -`lwaftr monitor` is a tool that helps monitorizing incoming and outgoing packets -in a lwAFTR. This feature must be combined with running SnabbVMX with mirroring +`lwaftr monitor` is a tool that helps monitoring incoming and outgoing packets +to a lwAFTR. This feature must be combined with running SnabbVMX with mirroring enabled. ```bash @@ -323,16 +324,16 @@ $ sudo ./snabb snabbvmx lwaftr --id xe0 --conf snabbvmx-lwaftr-xe0.cfg \ --pci 82:00.0 --mac 02:42:df:27:05:00 --mirror tap0 ``` -`mirror` parameter expects to find a tap interface. The interface must be up +The `mirror` parameter expects a tap interface. The interface must be up in order to receive packets: ```bash $ sudo ip link set dev tap0 up ``` -`lwaftr monitor` is set with a target IPv4 address. All lwAFTR packets which -IPv4 source match the target address will be mirrored in the tap interface. -In addition, monitor can be set with two special values: +`lwaftr monitor` is set with a target IPv4 address. All lwAFTR packets whose +IPv4 source matches the target address will be mirrored to the tap interface. +In addition, monitoring can be set with two special values: - `all`: mirrors all packets to the tap interface. - `none`: mirrors nothing to the tap interface. @@ -340,9 +341,9 @@ In addition, monitor can be set with two special values: ### Snabb's packetblaster lwaftr `snabb packetblaster` is a built-in Snabb utility which can blast packets to a -NIC very fast. It features several modes: `synth`, `replay` and `lwaftr`. +NIC very quickly. It features several modes: `synth`, `replay` and `lwaftr`. -The `lwaftr` mode is specially suited to generate IPv4 and IPv6 packets that +The `lwaftr` mode is especially suited to generating IPv4 and IPv6 packets that match a binding table. Example: ```bash @@ -356,13 +357,14 @@ Parameters: - `src_mac`: Source MAC-Address. - `dst_mac`: Destination MAC-Address. Must match a lwAFTR configuration file -MAC addresses. -- `b4`: IPV6,IPV4,PORT. IPV6, IPV4 and PORT values of the first softwire. -- `aftr`: IPv6 address of lwaftr server. Only one value can be specified. +MAC address. +- `b4`: IPV6, IPV4 and PORT values of the first softwire. +- `aftr`: IPv6 address of the lwaftr server. Only one value can be specified. - `count`: Number of B4 clients to simulate. - `rate`: Rate in MPPS for the generated traffic. - `pci`: Interface PCI address. -These are the most essential parameters to generate valid lwAFTR traffic to -a SnabbVMX instance. `packetblaster lwaftr` features much more flags and -options. Please refer to its Snabb's manual for a more detailed description. +These are the main parameters to generate valid lwAFTR traffic to a SnabbVMX +instance. `packetblaster lwaftr` features additional flags and options: +please refer to its section in the Snabb's manual for a more detailed +description. From faa32bfcacda682f19bdfabc5721fef3e3972a97 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 27 Oct 2016 13:32:26 +0200 Subject: [PATCH 061/631] Add image --- src/program/snabbvmx/doc/.images/snabbvmx.png | 1 + 1 file changed, 1 insertion(+) create mode 120000 src/program/snabbvmx/doc/.images/snabbvmx.png diff --git a/src/program/snabbvmx/doc/.images/snabbvmx.png b/src/program/snabbvmx/doc/.images/snabbvmx.png new file mode 120000 index 0000000000..8d5c7c5212 --- /dev/null +++ b/src/program/snabbvmx/doc/.images/snabbvmx.png @@ -0,0 +1 @@ +../../.images/snabbvmx.png \ No newline at end of file From db2e4369522aee5ec2ee2ec8391ac1d0bd7be7d8 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 27 Oct 2016 16:15:30 +0200 Subject: [PATCH 062/631] Rename snabb-lwaftr to snabbvmx in Makefile --- src/program/snabbvmx/Makefile | 18 +++++++++--------- src/program/snabbvmx/doc/.gitignore | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/program/snabbvmx/Makefile b/src/program/snabbvmx/Makefile index 978acc87c0..7b4a285349 100644 --- a/src/program/snabbvmx/Makefile +++ b/src/program/snabbvmx/Makefile @@ -1,24 +1,24 @@ Q= @ E= @echo -snabb-lwaftr-doc: doc/snabb-lwaftr.pdf doc/snabb-lwaftr.html doc/snabb-lwaftr.epub +snabbvmx-doc: doc/snabbvmx.pdf doc/snabbvmx.html doc/snabbvmx.epub -doc/snabb-lwaftr.md: +doc/snabbvmx.md: (cd doc; ./genbook.sh) > $@ -doc/snabb-lwaftr.pdf: doc/snabb-lwaftr.md +doc/snabbvmx.pdf: doc/snabbvmx.md $(E) "PANDOC $@" - $(Q) (cd doc; pandoc -S --toc --chapters -o snabb-lwaftr.pdf snabb-lwaftr.md) + $(Q) (cd doc; pandoc -S --toc --chapters -o snabbvmx.pdf snabbvmx.md) -doc/snabb-lwaftr.html: doc/snabb-lwaftr.md +doc/snabbvmx.html: doc/snabbvmx.md $(E) "PANDOC $@" - $(Q) (cd doc; pandoc --self-contained --css="../../../doc/style.css" -S -s --toc --chapters -o snabb-lwaftr.html snabb-lwaftr.md) + $(Q) (cd doc; pandoc --self-contained --css="../../../doc/style.css" -S -s --toc --chapters -o snabbvmx.html snabbvmx.md) -doc/snabb-lwaftr.epub: doc/snabb-lwaftr.md +doc/snabbvmx.epub: doc/snabbvmx.md $(E) "PANDOC $@" - $(Q) (cd doc; pandoc --self-contained --css="../../../doc/style.css" -S -s --toc --chapters -o snabb-lwaftr.epub snabb-lwaftr.md) + $(Q) (cd doc; pandoc --self-contained --css="../../../doc/style.css" -S -s --toc --chapters -o snabbvmx.epub snabbvmx.md) -CLEAN = doc/snabb-lwaftr.* +CLEAN = doc/snabbvmx.* clean: $(E) "RM $(CLEAN)" diff --git a/src/program/snabbvmx/doc/.gitignore b/src/program/snabbvmx/doc/.gitignore index 41750185b6..297a2f337b 100644 --- a/src/program/snabbvmx/doc/.gitignore +++ b/src/program/snabbvmx/doc/.gitignore @@ -1,4 +1,4 @@ -snabb-lwaftr.epub -snabb-lwaftr.html -snabb-lwaftr.md -snabb-lwaftr.pdf +snabbvmx.epub +snabbvmx.html +snabbvmx.md +snabbvmx.pdf From d3b1d95a0ebad103290823bcefb5486accb32aef Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 25 Oct 2016 14:01:35 +0000 Subject: [PATCH 063/631] Redo exit logic in snabbvmx selftest and next_hop --- src/program/snabbvmx/tests/nexthop/selftest.sh | 12 ++++-------- src/program/snabbvmx/tests/selftest.sh | 3 ++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index 094903fcf9..5134cad838 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -46,8 +46,9 @@ function quit_screens { } function cleanup { + local exit_code=$1 quit_screens - exit 0 + exit $exit_code } trap cleanup EXIT HUP INT QUIT TERM @@ -89,17 +90,12 @@ while true; do $(last_32bit "$mac_v6") == "99:99:99:99" ]]; then echo "Resolved MAC inet side: $mac_v4 [OK]" echo "Resolved MAC inet side: $mac_v6 [OK]" - break + exit 0 fi if [[ $count == $TIMEOUT ]]; then - break + exit 1 fi count=$((count + 1)) sleep 1 done - -if [[ $count == $TIMEOUT ]]; then - echo "Couldn't resolve nexthop" - exit 1 -fi diff --git a/src/program/snabbvmx/tests/selftest.sh b/src/program/snabbvmx/tests/selftest.sh index d2052032b6..030e685f0e 100755 --- a/src/program/snabbvmx/tests/selftest.sh +++ b/src/program/snabbvmx/tests/selftest.sh @@ -191,13 +191,14 @@ function check_pcap_equals { testname=$1; output=$2; expected=$3 } function cleanup { + local exit_code=$1 screens=$(screen -ls | egrep -o "[0-9]+\." | sed 's/\.//') for each in $screens; do if [[ "$each" > 0 ]]; then screen -S $each -X quit fi done - exit 0 + echo $exit_code } function run_pcap_test { testname=$1; input=$2; expected=$3; filter=$4 From 1d1c0c7935fb36851e2be316045710f18251a848 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 25 Oct 2016 15:49:21 +0200 Subject: [PATCH 064/631] Add unit test for ipv4_cache_trigger --- src/apps/lwaftr/nh_fwd.lua | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index 27a82fc22f..ff016c4dd5 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -2,6 +2,7 @@ module(..., package.seeall) local app = require("core.app") local basic_apps = require("apps.basic.basic_apps") +local bit = require("bit") local constants = require("apps.lwaftr.constants") local ethernet = require("lib.protocol.ethernet") local ipsum = require("lib.checksum").ipsum @@ -18,6 +19,7 @@ local transmit, receive = link.transmit, link.receive local htons = lib.htons local rd16, rd32, wr16 = lwutil.rd16, lwutil.rd32, lwutil.wr16 local ipv6_equals = lwutil.ipv6_equals +local lshift, band = bit.lshift, bit.band nh_fwd4 = { config = { @@ -119,22 +121,28 @@ local function send_ipv6_cache_trigger (r, pkt, mac) transmit(r, pkt) end -local function send_ipv4_cache_trigger(r, pkt, mac) - -- Set a bogus source IP address of 0.0.0.0. +local function ipv4_cache_trigger (pkt, mac) local ether_dhost = get_ether_dhost_ptr(pkt) local ipv4_hdr = get_ethernet_payload(pkt) - local ipv4_hdr_size = get_ipv4_header_length(ipv4_header) + local ipv4_hdr_size = get_ipv4_header_length(ipv4_hdr) local ipv4_src_ip = get_ipv4_src_ptr(ipv4_hdr) local ipv4_checksum = get_ipv4_checksum_ptr(ipv4_hdr) -- VM will discard packets not matching its MAC address on the interface. copy_ether(ether_dhost, mac) + + -- Set a bogus source IP address. copy_ipv4(ipv4_src_ip, n_cache_src_ipv4) -- Clear checksum to recalculate it with new source IPv4 address. wr16(ipv4_checksum, 0) wr16(ipv4_checksum, htons(ipsum(pkt.data + ethernet_header_size, ipv4_hdr_size, 0))) - transmit(r, pkt) + + return pkt +end + +local function send_ipv4_cache_trigger (r, pkt, mac) + transmit(r, ipv4_cache_trigger(pkt, mac)) end function nh_fwd4:new (conf) @@ -566,8 +574,23 @@ local function test_ipv6_flow () test_ipv6_service_to_vm({pkt1}) end +local function test_ipv4_cache_trigger () + local pkt = packet.from_string(lib.hexundump([[ + 02:aa:aa:aa:aa:aa 02:99:99:99:99:99 08 00 45 00 + 02 18 00 00 00 00 0f 11 d3 61 0a 0a 0a 01 c1 05 + 01 64 30 39 04 00 00 26 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 + ]], 72)) + local ether_dhost = "52:54:00:00:00:01" + local refresh_packet = ipv4_cache_trigger(pkt, ethernet:pton(ether_dhost)) + local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) + assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) +end + function selftest () print("nh_fwd: selftest") test_ipv4_flow() test_ipv6_flow() + test_ipv4_cache_trigger() end From 9856da7802247e70ba7410666d0048fc7a1e2f07 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 27 Oct 2016 14:48:40 +0000 Subject: [PATCH 065/631] Add unit test for ipv6_cache_trigger --- src/apps/lwaftr/nh_fwd.lua | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index ff016c4dd5..de50908b4b 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -110,15 +110,23 @@ end -- Using the link local address fe80::, the packets are properly routed back -- thru the same interface. Not sure if its OK to use that address or if there -- is a better way. -local function send_ipv6_cache_trigger (r, pkt, mac) +-- +local function ipv6_cache_trigger (pkt, mac) local ether_dhost = get_ether_dhost_ptr(pkt) local ipv6_hdr = get_ethernet_payload(pkt) local ipv6_src_ip = get_ipv6_src_address(ipv6_hdr) -- VM will discard packets not matching its MAC address on the interface. copy_ether(ether_dhost, mac) + + -- Set a bogus source IP address. copy_ipv6(ipv6_src_ip, n_cache_src_ipv6) - transmit(r, pkt) + + return pkt +end + +local function send_ipv6_cache_trigger (r, pkt, mac) + transmit(r, ipv6_cache_trigger(pkt, mac)) end local function ipv4_cache_trigger (pkt, mac) @@ -588,9 +596,25 @@ local function test_ipv4_cache_trigger () assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) end +local function test_ipv6_cache_trigger () + local pkt = packet.from_string(lib.hexundump([[ + 02:aa:aa:aa:aa:aa 02:99:99:99:99:99 86 dd 60 00 + 01 f0 01 f0 04 ff fc 00 00 01 00 02 00 03 00 04 + 00 05 00 00 00 7e fc 00 00 00 00 00 00 00 00 00 + 00 00 00 00 01 00 45 00 01 f0 00 00 00 00 0f 11 + d3 89 c1 05 01 64 0a 0a 0a 01 04 00 30 39 00 0c + 00 00 00 00 00 00 + ]], 86)) + local ether_dhost = "52:54:00:00:00:01" + local refresh_packet = ipv6_cache_trigger(pkt, ethernet:pton(ether_dhost)) + local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) + assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) +end + function selftest () print("nh_fwd: selftest") test_ipv4_flow() test_ipv6_flow() test_ipv4_cache_trigger() + test_ipv6_cache_trigger() end From 324d5616d1620d825b0cf12a80d154bcd479cfc6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 27 Oct 2016 17:09:42 +0200 Subject: [PATCH 066/631] Remove data-related facilities from schema2.lua --- src/lib/yang/schema2.lua | 65 ++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua index 32fef89936..fc5973e5a4 100644 --- a/src/lib/yang/schema2.lua +++ b/src/lib/yang/schema2.lua @@ -591,40 +591,16 @@ local function strip(exp) return ret end -local function integer_type(min, max) return function(node) return node end end -local function binary_type(node) return node end -local function bits_type(node) return node end -local function boolean_type(node) return node end -local function decimal64_type(node) return node end -local function empty_type(node) return node end -local function enumeration_type(node) return node end -local function identityref_type(node) return node end -local function instance_identifier_type(node) return node end -local function leafref_type(node) return node end -local function string_type(node) return node end -local function union_type(node) return node end - -local primitive_type_constructors = { - int8 = integer_type(-0xf0, 0x7f), - int16 = integer_type(-0xf000, 0x7fff), - int32 = integer_type(-0xf000000, 0x7fffffff), - int64 = integer_type(-0xf00000000000000LL, 0x7fffffffffffffffLL), - uint8 = integer_type(0, 0xff), - uint16 = integer_type(0, 0xffff), - uint32 = integer_type(0, 0xffffffff), - uint64 = integer_type(0, 0xffffffffffffffffULL), - binary = binary_type, - bits = bits_type, - boolean = boolean_type, - decimal64 = decimal64_type, - empty = empty_type, - enumeration = enumeration_type, - identityref = identityref_type, - ['instance-identifier'] = instance_identifier_type, - leafref = leafref_type, - string = string_type, - union = union_type -} +local function set(...) + local ret = {} + for k, v in pairs({...}) do ret[v] = true end + return ret +end + +local primitive_types = set( + 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', + 'binary', 'bits', 'boolean', 'decimal64', 'empty', 'enumeration', + 'identityref', 'instance-identifier', 'leafref', 'string', 'union') -- Inline "grouping" into "uses". -- Inline "submodule" into "include". @@ -671,16 +647,20 @@ function resolve(schema, features) local ret = {} local prefix = lookup(env, 'prefix', '_') local function error_recursion() - error('mutually recursive typedefs or groupings') end for k,v in pairs(tab) do -- FIXME: Only add prefix:k if at top level. + local state local function lazy() - ret[k] = error_recursion - ret[prefix..':'..k] = ret[k] - ret[k] = visit(v, env) - ret[prefix..':'..k] = ret[k] - return ret[k] + if state == 'visiting' then + error('mutually recursive definitions: '..k) + elseif state then + return state + else + state = 'visiting' + end + state = visit(v, env) + return state end ret[k] = lazy ret[prefix..':'..k] = ret[k] @@ -698,9 +678,8 @@ function resolve(schema, features) node.base_type = typedef else -- If the type name wasn't bound, it must be primitive. - local make_primitive = assert(primitive_type_constructors[node.id], - 'unknown type: '..node.id) - node.primitive_type = make_primitive(node) + assert(primitive_types[node.id], 'unknown type: '..node.id) + node.base_type = node.id end return node end From 6ce76cf412d069f3125f79c3c58f4a8ff80b2a82 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 27 Oct 2016 17:10:04 +0200 Subject: [PATCH 067/631] Add YANG data parser module This generates a specific data parser from a YANG schema. Next up, a compiler and a loader for the compiled format. --- src/lib/yang/data.lua | 290 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 src/lib/yang/data.lua diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua new file mode 100644 index 0000000000..8f9fa77d99 --- /dev/null +++ b/src/lib/yang/data.lua @@ -0,0 +1,290 @@ +-- Use of this source code is governed by the Apache 2.0 license; see +-- COPYING. +module(..., package.seeall) + +local ffi = require("ffi") +local parser = require("lib.yang.parser") +local schema = require("lib.yang.schema2") + +ffi.cdef([[ +unsigned long long strtoull (const char *nptr, const char **endptr, int base); +]]) + +local function integer_type(min, max) + return function(str, k) + local str = assert(str, 'missing value for '..k) + local start = 1 + local is_negative + local base = 10 + if str:match('^-') then start, is_negative = 2, true + elseif str:match('^+') then start = 2 end + if str:match('^0x', start) then base, start = 16, start + 2 + elseif str:match('^0', start) then base = 8 end + str = str:lower() + local function check(test) + return assert(test, 'invalid numeric value for '..k..': '..str) + end + check(start <= str:len()) + -- FIXME: check that res did not overflow the 64-bit number + local res = ffi.C.strtoull(str:sub(start), nil, base) + if is_negative then + res = ffi.new('int64_t[1]', -1*res)[0] + check(res <= 0) + end + check(min <= res and res <= max) + if tonumber(res) == res then return tonumber(res) end + return res + end +end + +local primitive_parsers = {} + +primitive_parsers.int8 = integer_type(-0xf0, 0x7f) +primitive_parsers.int16 = integer_type(-0xf000, 0x7fff) +primitive_parsers.int32 = integer_type(-0xf000000, 0x7fffffff) +primitive_parsers.int64 = integer_type(-0xf00000000000000LL, 0x7fffffffffffffffLL) +primitive_parsers.uint8 = integer_type(0, 0xff) +primitive_parsers.uint16 = integer_type(0, 0xffff) +primitive_parsers.uint32 = integer_type(0, 0xffffffff) +primitive_parsers.uint64 = integer_type(0, 0xffffffffffffffffULL) +function primitive_parsers.binary(str, k) + error('unimplemented: binary') + return str +end +function primitive_parsers.bits(str, k) + error('unimplemented: bits') +end +function primitive_parsers.boolean(str, k) + local str = assert(str, 'missing value for '..k) + if str == 'true' then return true end + if str == 'false' then return false end + error('bad boolean value: '..str) +end +function primitive_parsers.decimal64(str, k) + error('unimplemented: decimal64') +end +function primitive_parsers.empty(str, k) + assert(not str, 'unexpected value for '..k) + return true +end +function primitive_parsers.enumeration(str, k) + return assert(str, 'missing value for '..k) +end +function primitive_parsers.identityref(str, k) + error('unimplemented: identityref') +end +primitive_parsers['instance-identifier'] = function(str, k) + error('unimplemented: instance-identifier') +end +function primitive_parsers.leafref(str, k) + error('unimplemented: leafref') +end +function primitive_parsers.string(str, k) + return assert(str, 'missing value for '..k) +end +function primitive_parsers.union(str, k) + error('unimplemented: union') +end + +-- FIXME :) +local function range_validator(range, f) return f end +local function length_validator(range, f) return f end +local function pattern_validator(range, f) return f end +local function bit_validator(range, f) return f end +local function enum_validator(range, f) return f end + +local function value_parser(typ) + local prim = typ.base_type + -- FIXME: perhaps cache the primitive type on all type nodes. + while type(prim) ~= 'string' do prim = prim.base_type end + local parse = assert(primitive_parsers[prim], prim) + local function validate(val) end + validate = range_validator(typ.range, validate) + validate = length_validator(typ.length, validate) + validate = pattern_validator(typ.pattern, validate) + validate = bit_validator(typ.bit, validate) + validate = enum_validator(typ.enum, validate) + -- TODO: union, require-instance. + return function(str, k) + local val = parse(str, k) + validate(val) + return val + end +end + +local function struct_parser(members, k) + local function init() return nil end + local function parse1(node) + assert(not node.argument, 'argument unexpected for struct type: '..k) + assert(node.statements, 'missing statements for struct type: '..k) + local ret = {} + for k,sub in pairs(members) do ret[k] = sub.init() end + for _,node in ipairs(node.statements) do + local sub = assert(members[node.keyword], + 'unrecognized keyword: '..node.keyword) + ret[node.keyword] = sub.parse(node, ret[node.keyword]) + end + for k,sub in pairs(members) do ret[k] = sub.finish(ret[k]) end + return ret + end + local function parse(node, out) + if out then error('duplicate struct: '..k) end + return parse1(node) + end + local function finish(out) + -- FIXME check mandatory + return out + end + return {init=init, parse=parse, finish=finish} +end + +local function array_parser(typ, k) + local function init() return {} end + local parsev = value_parser(typ) + local function parse1(node) + assert(node.argument, 'argument expected for array type: '..k) + assert(not node.statements, 'unexpected statements for array type: '..k) + return parsev(node.argument, k) + end + local function parse(node, out) + table.insert(out, parse1(node)) + return out + end + local function finish(out) + -- FIXME check min-elements + return out + end + return {init=init, parse=parse, finish=finish} +end + +local function scalar_parser(typ, default, mandatory, k) + local function init() return nil end + local parsev = value_parser(typ) + local function parse1(node) + assert(not node.statements, 'unexpected statements for scalar type: '..k) + return parsev(node.argument, k) + end + local function parse(node, out) + assert(not out, 'duplicate scalar config value: '..k) + return parse1(node) + end + local function finish(out) + if out ~= nil then return out end + if default then return parse1(default) end + if mandatory then error('missing scalar value: '..k) end + end + return {init=init, parse=parse, finish=finish} +end + +local function table_parser(keystr, members, k) + -- This is a temporary lookup until we get the various Table kinds + -- working. + local function lookup(out, k) + for _,v in ipairs(out) do + if v[keystr] == k then return v end + end + error('not found: '..k) + end + local function init() return {lookup=lookup} end + local parser = struct_parser(members, k) + local function parse1(node) + assert(not node.argument, 'argument unexpected for table type: '..k) + assert(node.statements, 'expected statements for table type: '..k) + return parser.finish(parser.parse(node, parser.init())) + end + local function parse(node, out) + -- TODO: tease out key from value, add to associative array + table.insert(out, parse1(node)) + return out + end + local function finish(out) + -- FIXME check min-elements + return out + end + return {init=init, parse=parse, finish=finish} +end + +function data_parser_from_schema(schema) + local handlers = {} + local function visit(node) + local handler = handlers[node.kind] + if handler then return handler(node) end + return {} + end + local function visit_body(node) + local ret = {} + for id,node in pairs(node.body) do + for keyword,node in pairs(visit(node)) do + assert(not ret[keyword], 'duplicate identifier: '..keyword) + ret[keyword] = node + end + end + return ret + end + function handlers.module(node) + end + function handlers.container(node) + if not node.presence then return visit_body(node) end + return {[node.id]=struct_parser(visit_body(node), node.id)} + end + handlers['leaf-list'] = function(node) + return {[node.id]=array_parser(node.type, node.id)} + end + function handlers.list(node) + return {[node.id]=table_parser(node.key, visit_body(node), node.id)} + end + function handlers.leaf(node) + return {[node.id]=scalar_parser(node.type, node.default, node.mandatory, +node.id)} + end + + local parser = struct_parser(visit_body(schema), '(top level)') + return function(stmtlist) + return parser.finish(parser.parse({statements=stmtlist}, parser.init())) + end +end + +function selftest() + local test_schema = [[module fruit { + namespace "urn:testing:fruit"; + prefix "fruit"; + grouping fruit { + leaf name { + type string; + mandatory true; + } + leaf score { + type uint8 { range 0..10; } + mandatory true; + } + leaf tree-grown { type boolean; } + } + + container fruit-bowl { + presence true; + leaf description { type string; } + list contents { uses fruit; key name; } + } + }]] + + local schema = schema.load_schema(test_schema) + local parse = data_parser_from_schema(schema) + local data = parse(parser.parse_string([[ + fruit-bowl { + description 'ohai'; + contents { name foo; score 7; } + contents { name bar; score 8; } + contents { name baz; score 9; tree-grown true; } + } + ]])) + assert(data['fruit-bowl'].description == 'ohai') + assert(data['fruit-bowl'].contents:lookup('foo').name == 'foo') + assert(data['fruit-bowl'].contents:lookup('foo').score == 7) + assert(data['fruit-bowl'].contents:lookup('foo')['tree-grown'] == nil) + assert(data['fruit-bowl'].contents:lookup('bar').name == 'bar') + assert(data['fruit-bowl'].contents:lookup('bar').score == 8) + assert(data['fruit-bowl'].contents:lookup('bar')['tree-grown'] == nil) + assert(data['fruit-bowl'].contents:lookup('baz').name == 'baz') + assert(data['fruit-bowl'].contents:lookup('baz').score == 9) + assert(data['fruit-bowl'].contents:lookup('baz')['tree-grown'] == true) +end From ccce15f29df7c07164e180505147a94f32c1c547 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 27 Oct 2016 09:08:27 +0000 Subject: [PATCH 068/631] Rewrite of snabbvmx's nexthop selftest.sh --- .../snabbvmx/tests/nexthop/selftest.sh | 50 ++++++----------- .../snabbvmx/tests/test_env/test_env.sh | 54 ++++++++----------- 2 files changed, 36 insertions(+), 68 deletions(-) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index 5134cad838..36829b35f1 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -20,7 +20,6 @@ fi LWAFTR_IPV4_ADDRESS=10.0.1.1 LWAFTR_IPV6_ADDRESS=fc00::100 MAC_ADDRESS_NET0=02:AA:AA:AA:AA:AA -MIRROR_TAP=tap0 NEXT_HOP_MAC=02:99:99:99:99:99 NEXT_HOP_V4=10.0.1.100 NEXT_HOP_V6=fc00::1 @@ -29,53 +28,31 @@ PCAP_INPUT=$SNABBVMX_DIR/tests/pcap/input PCAP_OUTPUT=$SNABBVMX_DIR/tests/pcap/output SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr.cfg SNABBVMX_ID=xe1 -SNABBVMX_LOG=snabbvmx.log SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock -# Load environment settings. -source program/snabbvmx/tests/test_env/test_env.sh - -function quit_screens { - screens=$(screen -ls | egrep -o "[0-9]+\." | sed 's/\.//') - for each in $screens; do - if [[ "$each" > 0 ]]; then - screen -S $each -X quit - fi - done +function last_32bit { + mac=$1 + echo `echo $mac | egrep -o "[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+$"` } function cleanup { - local exit_code=$1 - quit_screens - exit $exit_code + exit $1 } trap cleanup EXIT HUP INT QUIT TERM -# Override settings. -SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr-xe0.cfg -TARGET_MAC_INET=02:99:99:99:99:99 -TARGET_MAC_B4=02:99:99:99:99:99 - -# Clean up log file. -rm -f $SNABBVMX_LOG - -# Run SnabbVMX. -cmd="./snabb snabbvmx lwaftr --conf $SNABBVMX_CONF --id $SNABBVMX_ID --pci $SNABB_PCI0 --mac $MAC_ADDRESS_NET0 --sock $VHU_SOCK0" -run_cmd_in_screen "snabbvmx" "$cmd" +# Import SnabbVMX test_env. +if ! source program/snabbvmx/tests/test_env/test_env.sh; then + echo "Could not load snabbvmx test_env."; exit 1 +fi -# Run QEMU. +# Main. start_test_env -# Flush lwAFTR packets to SnabbVMX. -cmd="./snabb packetblaster replay -D 10 $PCAP_INPUT/v4v6-256.pcap $SNABB_PCI1" -run_cmd_in_screen "packetblaster" "$cmd" - -function last_32bit { - mac=$1 - echo `echo $mac | egrep -o "[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+$"` -} +if ! snabb $SNABB_PCI1 "packetblaster replay -D 10 $PCAP_INPUT/v4v6-256.pcap $SNABB_PCI1"; then + echo "Could not run packetblaster" +fi # Query nexthop for 10 seconds. TIMEOUT=10 @@ -94,8 +71,11 @@ while true; do fi if [[ $count == $TIMEOUT ]]; then + echo "Could not resolve nexthop" exit 1 fi + + # Try again until TIMEOUT. count=$((count + 1)) sleep 1 done diff --git a/src/program/snabbvmx/tests/test_env/test_env.sh b/src/program/snabbvmx/tests/test_env/test_env.sh index c4b5ebed6e..2d7f7bee61 100755 --- a/src/program/snabbvmx/tests/test_env/test_env.sh +++ b/src/program/snabbvmx/tests/test_env/test_env.sh @@ -7,11 +7,6 @@ if [[ $EUID != 0 ]]; then exit 1 fi -BZ_IMAGE="$HOME/.test_env/bzImage" -HUGEPAGES_FS=/dev/hugepages -IMAGE="$HOME/.test_env/qemu.img" -MEM=1024M - function run_telnet { (echo "$2"; sleep ${3:-2}) \ | telnet localhost $1 2>&1 @@ -36,47 +31,40 @@ function wait_vm_up { echo " [OK]" } -function qemu_cmd { - echo "qemu-system-x86_64 \ - -kernel ${BZ_IMAGE} -append \"earlyprintk root=/dev/vda rw console=tty0\" \ - -enable-kvm -drive format=raw,if=virtio,file=${IMAGE} \ - -M pc -smp 1 -cpu host -m ${MEM} \ - -object memory-backend-file,id=mem,size=${MEM},mem-path=${HUGEPAGES_FS},share=on \ - -numa node,memdev=mem \ - -chardev socket,id=char1,path=${VHU_SOCK0},server \ - -netdev type=vhost-user,id=net0,chardev=char1 \ - -device virtio-net-pci,netdev=net0,addr=0x8,mac=${MAC_ADDRESS_NET0} \ - -serial telnet:localhost:${SNABB_TELNET0},server,nowait \ - -display none" -} +# Define vars before importing SnabbNFV test_env to default initialization. +MAC=$MAC_ADDRESS_NET0 +IP=$LWAFTR_IPV6_ADDRESS -function quit_screen { screen_id=$1 - screen -X -S "$screen_id" quit &> /dev/null -} +if ! source program/snabbnfv/test_env/test_env.sh; then + echo "Could not load snabbnfv test_env."; exit 1 +fi -function run_cmd_in_screen { screen_id=$1; cmd=$2 - screen_id="${screen_id}-$$" - quit_screen "$screen_id" - screen -dmS "$screen_id" bash -c "$cmd >> $SNABBVMX_LOG" +# Overwrite mac function to always return $MAC. +function mac { + echo $MAC } -function qemu { - run_cmd_in_screen "qemu" "`qemu_cmd`" +# Overwrite ip function to always return $IP. +function ip { + echo $IP } function start_test_env { - if [[ ! -f "$IMAGE" ]]; then - echo "Couldn't find QEMU image: $IMAGE" - exit $SKIPPED_CODE + local cmd="snabbvmx lwaftr --conf $SNABBVMX_CONF --id $SNABBVMX_ID --pci $SNABB_PCI0 --mac $MAC_ADDRESS_NET0 --sock $VHU_SOCK0" + if ! snabb $SNABB_PCI0 "$cmd"; then + echo "Could not start snabbvmx."; exit 1 fi - # Run qemu. - qemu + if ! qemu $SNABB_PCI0 $VHU_SOCK0 $SNABB_TELNET0 $MAC_ADDRESS_NET0; then + echo "Could not start qemu 0."; exit 1 + fi # Wait until VMs are ready. wait_vm_up $SNABB_TELNET0 - # Manually set ip addresses. + # Configure eth0 interface in the VM. + + # Bring up interface. run_telnet $SNABB_TELNET0 "ifconfig eth0 up" >/dev/null # Assign lwAFTR's IPV4 and IPV6 addresses to eth0. From c404ea67f3df555f9bc1c20d195ec1e9a5eb5c13 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 27 Oct 2016 20:50:18 +0000 Subject: [PATCH 069/631] Rewrite SnabbVMX's selftest.sh --- src/program/snabbvmx/tests/selftest.sh | 121 +++++++++--------- .../snabbvmx/tests/test_env/test_env.sh | 6 + 2 files changed, 69 insertions(+), 58 deletions(-) diff --git a/src/program/snabbvmx/tests/selftest.sh b/src/program/snabbvmx/tests/selftest.sh index 030e685f0e..f580cf4a78 100755 --- a/src/program/snabbvmx/tests/selftest.sh +++ b/src/program/snabbvmx/tests/selftest.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# set -x + SKIPPED_CODE=43 if [[ $EUID != 0 ]]; then @@ -8,10 +10,12 @@ if [[ $EUID != 0 ]]; then fi if [[ -z "$SNABB_PCI0" ]]; then + echo "Skip test: SNABB_PCI0 not defined" exit $SKIPPED_CODE fi if [[ -z "$SNABB_PCI1" ]]; then + echo "Skip test: SNABB_PCI1 not defined" exit $SKIPPED_CODE fi @@ -27,51 +31,36 @@ PCAP_INPUT=$SNABBVMX_DIR/tests/pcap/input PCAP_OUTPUT=$SNABBVMX_DIR/tests/pcap/output SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr.cfg SNABBVMX_ID=xe1 -SNABBVMX_LOG=snabbvmx.log SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock -# Load environment settings. -source program/snabbvmx/tests/test_env/test_env.sh - -rm -f $SNABBVMX_LOG - function monitor { action=$1 - local cmd="sudo ./snabb lwaftr monitor $action" - run_cmd_in_screen "lwaftr-monitor" "$cmd" + ./snabb lwaftr monitor $action &> /dev/null } -function tcpreplay { pcap=$1; pci=$2 - local cmd="sudo ./snabb packetblaster replay --no-loop $pcap $pci" - run_cmd_in_screen "tcpreplay" "$cmd" +function tcpreplay { + pcap=$1; pci=$2 + snabb $pci "packetblaster replay --no-loop $pcap $pci" } function create_mirror_tap_if_needed { - ip tuntap add $MIRROR_TAP mode tap 2>/dev/null - ip li set dev $MIRROR_TAP up 2>/dev/null - ip li sh $MIRROR_TAP &>/dev/null + sudo ip tuntap add $MIRROR_TAP mode tap &>/dev/null + sudo ip li set dev $MIRROR_TAP up &>/dev/null + sudo ip li sh $MIRROR_TAP &>/dev/null if [[ $? -ne 0 ]]; then echo "Couldn't create mirror tap: $MIRROR_TAP" exit 1 fi } -function run_snabbvmx { - echo "Launch Snabbvmx" - local cmd="./snabb snabbvmx lwaftr --conf $SNABBVMX_CONF --id $SNABBVMX_ID \ - --pci $SNABB_PCI0 --mac $MAC_ADDRESS_NET0 --sock $VHU_SOCK0 \ - --mirror $MIRROR_TAP " - run_cmd_in_screen "snabbvmx" "$cmd" -} - -function capture_mirror_tap_to_file { fileout=$1; filter=$2 - local cmd="" +function capture_mirror_tap_to_file { + fileout=$1; filter=$2 if [[ -n $filter ]]; then - cmd="sudo tcpdump \"${filter}\" -U -c 1 -i $MIRROR_TAP -w $fileout" + tmux_launch "tcpdump" "tcpdump \"${filter}\" -U -c 1 -i $MIRROR_TAP -w $fileout" else - cmd="sudo tcpdump -U -c 1 -i $MIRROR_TAP -w $fileout" + tmux_launch "tcpdump" "tcpdump -U -c 1 -i $MIRROR_TAP -w $fileout" fi - run_cmd_in_screen "tcpdump" "$cmd" + count=$((count + 1)) } function myseq { from=$1; to=$2 @@ -122,9 +111,12 @@ function zero_checksum { file=$1; row=$2; column=$3 done } +function filesize { filename=$1 + echo $(ls -l $filename | awk '{ print $5 }') +} + function pcap2text { pcap=$1; txt=$2 - filesize=$(ls -l $pcap | awk '{ print $5 }') - if [[ $filesize < 40 ]]; then + if [[ $(filesize $pcap) < 40 ]]; then # Empty file. rm -f $txt touch $txt @@ -134,17 +126,25 @@ function pcap2text { pcap=$1; txt=$2 } function icmpv4_cmp { pcap1=$1; pcap2=$2 - local actual=/tmp/actual.txt - local expected=/tmp/expected.txt + local ret=0 + + # Compare filesize. + if [[ $(filesize $pcap1) != $(filesize $pcap2) ]]; then + ret=1 + else + local actual=/tmp/actual.txt + local expected=/tmp/expected.txt - pcap2text $pcap1 $actual - pcap2text $pcap2 $expected + pcap2text $pcap1 $actual + pcap2text $pcap2 $expected - zero_identifier $actual $expected - zero_checksum $actual $expected + zero_identifier $actual $expected + zero_checksum $actual $expected - local out=$(diff $actual $expected) - echo ${#out} + local out=$(diff $actual $expected) + ret=${#out} + fi + echo $ret } function check_icmpv4_equals { testname=$1; output=$2; expected=$3 @@ -168,14 +168,20 @@ function run_icmpv4_test { testname=$1; input=$2; expected=$3; filter=$4 } function pcap_cmp { pcap1=$1; pcap2=$2 - local actual=/tmp/actual.txt - local expected=/tmp/expected.txt + local ret=0 + if [[ $(filesize $pcap1) != $(filesize $pcap2) ]]; then + ret=1 + else + local actual=/tmp/actual.txt + local expected=/tmp/expected.txt - pcap2text $pcap1 $actual - pcap2text $pcap2 $expected + pcap2text $pcap1 $actual + pcap2text $pcap2 $expected - local out=$(diff $actual $expected) - echo ${#out} + local out=$(diff $actual $expected) + ret=${#out} + fi + echo $ret } function check_pcap_equals { testname=$1; output=$2; expected=$3 @@ -190,17 +196,6 @@ function check_pcap_equals { testname=$1; output=$2; expected=$3 fi } -function cleanup { - local exit_code=$1 - screens=$(screen -ls | egrep -o "[0-9]+\." | sed 's/\.//') - for each in $screens; do - if [[ "$each" > 0 ]]; then - screen -S $each -X quit - fi - done - echo $exit_code -} - function run_pcap_test { testname=$1; input=$2; expected=$3; filter=$4 local output="/tmp/output.pcap" capture_mirror_tap_to_file $output "$filter" @@ -273,17 +268,27 @@ function test_ndp_request_to_lwaftr { "$PCAP_OUTPUT/empty.pcap" } -# Set up graceful `exit'. +function cleanup { + exit $1 +} + trap cleanup EXIT HUP INT QUIT TERM -# Run snabbvmx with VM. +# Import SnabbVMX test_env. +if ! source program/snabbvmx/tests/test_env/test_env.sh; then + echo "Could not load snabbvmx test_env."; exit 1 +fi + +# Main. + +# Run SnabbVMX with VM. create_mirror_tap_if_needed -run_snabbvmx -start_test_env +start_test_env $MIRROR_TAP # Mirror all packets to tap0. monitor all +# Run tests. test_ping_to_lwaftr_inet test_ping_to_lwaftr_b4 test_arp_request_to_lwaftr diff --git a/src/program/snabbvmx/tests/test_env/test_env.sh b/src/program/snabbvmx/tests/test_env/test_env.sh index 2d7f7bee61..5dac08fec3 100755 --- a/src/program/snabbvmx/tests/test_env/test_env.sh +++ b/src/program/snabbvmx/tests/test_env/test_env.sh @@ -50,7 +50,13 @@ function ip { } function start_test_env { + local mirror=$1 + local cmd="snabbvmx lwaftr --conf $SNABBVMX_CONF --id $SNABBVMX_ID --pci $SNABB_PCI0 --mac $MAC_ADDRESS_NET0 --sock $VHU_SOCK0" + if [ -n "$mirror" ]; then + cmd="$cmd --mirror $mirror" + fi + if ! snabb $SNABB_PCI0 "$cmd"; then echo "Could not start snabbvmx."; exit 1 fi From 8b80812820c905035891e886188811584b07c5b2 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Fri, 28 Oct 2016 10:55:33 +0200 Subject: [PATCH 070/631] Fix the bench-file option in the run, run_nohw and transient commands (#524) --- src/program/lwaftr/run/run.lua | 2 +- src/program/lwaftr/run_nohw/run_nohw.lua | 2 +- src/program/lwaftr/transient/transient.lua | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 04737d89e7..04b0cd99ad 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -157,7 +157,7 @@ function run(args) end if opts.verbosity >= 1 then - local csv = csv_stats.CSVStatsTimer.new(opts.csv_file, opts.hydra) + local csv = csv_stats.CSVStatsTimer.new(opts.bench_file, opts.hydra) -- Why are the names cross-referenced like this? local ipv4_tx = opts.hydra and 'ipv4rx' or 'IPv4 RX' local ipv4_rx = opts.hydra and 'ipv4tx' or 'IPv4 TX' diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index 8d8cff4245..2a62e9dbdc 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -44,7 +44,7 @@ local function parse_args(args) lib.dogetopt(args, handlers, "b:c:B:I:vh", { help = "h", conf = "c", verbose = "v", ["b4-if"] = "B", ["inet-if"] = "I", - bench_file = 0, + ["bench-file"] = 0, }) check(conf_file, "no configuration specified (--conf/-c)") check(b4_if, "no B4-side interface specified (--b4-if/-B)") diff --git a/src/program/lwaftr/transient/transient.lua b/src/program/lwaftr/transient/transient.lua index d3813727f8..639a0db821 100644 --- a/src/program/lwaftr/transient/transient.lua +++ b/src/program/lwaftr/transient/transient.lua @@ -126,7 +126,7 @@ function run(args) rate_adjuster() timer.activate(timer.new("adjust_rate", rate_adjuster, opts.duration * 1e9, 'repeating')) - local csv = csv_stats.CSVStatsTimer.new(opts.csv_file) + local csv = csv_stats.CSVStatsTimer.new(opts.bench_file) for _,stream in ipairs(streams) do csv:add_app(stream.nic_id, { 'rx', 'tx' }, { rx=stream.name..' TX', tx=stream.name..' RX' }) From 90a6adaf7f7df47aba70acf9ce4d9699a6b2514b Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 28 Oct 2016 10:36:45 +0200 Subject: [PATCH 071/631] Replace usage of syscall.util and fix cleanup code This replaces the use of syscall.util so that core.app no longer uses that. It also fixes the supervisor which wasn't cleaning up thhe named app symlinks - it now does. --- src/core/app.lua | 8 +++++--- src/core/main.lua | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 820dc40d0f..22ef8b42a0 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -169,16 +169,18 @@ end -- This returns a table programs with the key being the name of the program -- and the value being the PID of the program. Each program is checked that -- it's still alive. Any dead program or program without a name is not listed. -function enumerate_named_programs() +-- +-- The inc_dead argument is to disable the alive check (default: false) +function enumerate_named_programs(inc_dead) local progs = {} - local dirs = S.util.dirtable(named_program_root, true) + local dirs = shm.children("/by-name") if dirs == nil then return progs end for _, program in pairs(dirs) do local fq = named_program_root .. "/" .. program local piddir = S.readlink(fq) local s, e = string.find(piddir, "/[^/]*$") local pid = tonumber(string.sub(piddir, s+1, e)) - if S.kill(pid, 0) then + if inc_dead == true or S.kill(pid, 0) then local ps, pe = string.find(fq, "/[^/]*$") local program_name = string.sub(fq, ps+1, pe) progs[program_name] = pid diff --git a/src/core/main.lua b/src/core/main.lua index a0a6d361f7..e03a156c18 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -146,9 +146,10 @@ function shutdown (pid) shm.unlink("/"..pid) -- Look through the named apps and unlink any which are for this process. - local progs = app.enumerate_named_programs() + local npid = tonumber(pid) + local progs = app.enumerate_named_programs(true) for name, p in pairs(progs) do - if p == pid then + if p == npid then S.unlink(app.named_program_root .. "/" .. name) end end From 0e9d54887a3b35abafa4167801d4170a018ab2d1 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sun, 30 Oct 2016 21:35:30 +0000 Subject: [PATCH 072/631] Allow duration in soaktest.sh --- src/program/lwaftr/soaktest/soaktest.lua | 2 +- src/program/lwaftr/tests/soaktest/core-soaktest.sh | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/program/lwaftr/soaktest/soaktest.lua b/src/program/lwaftr/soaktest/soaktest.lua index f41cd373ec..ad4106288d 100644 --- a/src/program/lwaftr/soaktest/soaktest.lua +++ b/src/program/lwaftr/soaktest/soaktest.lua @@ -21,7 +21,7 @@ function parse_args (args) local opts = {} function handlers.h() show_usage(0) end function handlers.D (arg) - opts.duration = tonumber(arg, "Duration must be a number") + opts.duration = assert(tonumber(arg), "Duration must be a number") end handlers["on-a-stick"] = function () opts["on-a-stick"] = true diff --git a/src/program/lwaftr/tests/soaktest/core-soaktest.sh b/src/program/lwaftr/tests/soaktest/core-soaktest.sh index a80d1f22e3..5ca6ac2a00 100755 --- a/src/program/lwaftr/tests/soaktest/core-soaktest.sh +++ b/src/program/lwaftr/tests/soaktest/core-soaktest.sh @@ -5,6 +5,8 @@ if [[ $EUID -ne 0 ]]; then exit 1 fi +DURATION=${1:-"0.10"} + function quit_with_msg { errno=$1; msg="$2" echo "Test failed: $msg" @@ -13,10 +15,10 @@ function quit_with_msg { function soaktest { conf="$1"; in_v4="$2"; in_v6="$3" - $SNABB_LWAFTR soaktest "$conf" "$in_v4" "$in_v6" || - quit_with_msg $? "Test failed: $SNABB_LWAFTR soaktest $@" - $SNABB_LWAFTR soaktest --on-a-stick "$conf" "$in_v4" "$in_v6" || - quit_with_msg $? "Test failed: $SNABB_LWAFTR soaktest --on-a-stick $@" + $SNABB_LWAFTR soaktest -D $DURATION "$conf" "$in_v4" "$in_v6" || + quit_with_msg $? "$SNABB_LWAFTR soaktest -D $DURATION $conf $in_v4 $in_v6" + $SNABB_LWAFTR soaktest -D $DURATION --on-a-stick "$conf" "$in_v4" "$in_v6" || + quit_with_msg $? "$SNABB_LWAFTR soaktest -D $DURATION --on-a-stick $conf $in_v4 $in_v6" } source "../end-to-end/test_env.sh" From 2729db908b412c3a43759754615d9862da655792 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sun, 30 Oct 2016 21:35:46 +0000 Subject: [PATCH 073/631] Drop origin_packet always --- src/apps/lwaftr/lwaftr.lua | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 0c97708efb..4986149b80 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -210,14 +210,15 @@ local function init_transmit_icmpv4_reply (lwstate) last_time = now num_packets = 0 end + -- Origin packet is always dropped. + if orig_pkt_link then + drop_ipv4(lwstate, orig_pkt, orig_pkt_link) + else + drop(orig_pkt) + end -- Send packet if limit not reached. if num_packets < icmpv4_rate_limiter_n_packets then num_packets = num_packets + 1 - if orig_pkt_link then - drop_ipv4(lwstate, orig_pkt, orig_pkt_link) - else - drop(orig_pkt) - end counter.add(lwstate.counters["out-icmpv4-bytes"], pkt.length) counter.add(lwstate.counters["out-icmpv4-packets"]) -- Only locally generated error packets are handled here. We transmit From 8d43cf29d89900c31aaa25f2f5e3b36c73ba7cb5 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 31 Oct 2016 09:19:19 +0100 Subject: [PATCH 074/631] Fix benchmark CSV creation and docs (#526) Correctly invoke the CSVStatsTimer constructor in benchmarking commands, add a short -b option for --bench-file to the run and run_nohw commands --- src/program/lwaftr/bench/README | 2 +- src/program/lwaftr/run/README | 4 ++-- src/program/lwaftr/run/run.lua | 8 +++----- src/program/lwaftr/run_nohw/README | 2 +- src/program/lwaftr/run_nohw/run_nohw.lua | 6 +++--- src/program/lwaftr/transient/transient.lua | 2 +- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/program/lwaftr/bench/README b/src/program/lwaftr/bench/README index fd007fb2e8..db7d4b208a 100644 --- a/src/program/lwaftr/bench/README +++ b/src/program/lwaftr/bench/README @@ -2,7 +2,7 @@ Usage: bench CONF IPV4-IN.PCAP IPV6-IN.PCAP -h, --help Print usage information. - -y HYDRA, --hydra + -y, --hydra Hydra mode: emit CSV data in the format expected by the Hydra reports. For instance: diff --git a/src/program/lwaftr/run/README b/src/program/lwaftr/run/README index 386a05d2e3..8d0bfd7984 100644 --- a/src/program/lwaftr/run/README +++ b/src/program/lwaftr/run/README @@ -17,7 +17,7 @@ Optional arguments: address set by "lwaftr monitor". -D Duration in seconds -v Verbose (repeat for more verbosity) - -y HYDRA, --hydra + -y, --hydra Hydra mode: emit CSV data in the format expected by the Hydra reports. For instance: @@ -26,7 +26,7 @@ Optional arguments: rather than the default: Time (s),Decap. MPPS,Decap. Gbps,Encap. MPPS,Encap. Gbps - --bench-file FILENAME + -b FILENAME, --bench-file FILENAME The file or path name to which benchmark data is written. A simple filename or relative pathname will be based on the current directory. Default diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 04b0cd99ad..9294a8c5e9 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -82,9 +82,7 @@ function parse_args(args) opts["mirror"] = ifname end function handlers.y() opts.hydra = true end - handlers["bench-file"] = function (bench_file) - opts.bench_file = bench_file - end + function handlers.b(arg) opts.bench_file = arg end handlers["ingress-drop-monitor"] = function (arg) if arg == 'flush' or arg == 'warn' then opts.ingress_drop_monitor = arg @@ -99,7 +97,7 @@ function parse_args(args) lib.dogetopt(args, handlers, "b:c:vD:yhir:", { conf = "c", v4 = 1, v6 = 1, ["v4-pci"] = 1, ["v6-pci"] = 1, verbose = "v", duration = "D", help = "h", virtio = "i", cpu = 1, - ["ring-buffer-size"] = "r", ["real-time"] = 0, ["bench-file"] = 0, + ["ring-buffer-size"] = "r", ["real-time"] = 0, ["bench-file"] = "b", ["ingress-drop-monitor"] = 1, ["on-a-stick"] = 1, mirror = 1, hydra = "y" }) if ring_buffer_size ~= nil then if opts.virtio_net then @@ -157,7 +155,7 @@ function run(args) end if opts.verbosity >= 1 then - local csv = csv_stats.CSVStatsTimer.new(opts.bench_file, opts.hydra) + local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) -- Why are the names cross-referenced like this? local ipv4_tx = opts.hydra and 'ipv4rx' or 'IPv4 RX' local ipv4_rx = opts.hydra and 'ipv4tx' or 'IPv4 TX' diff --git a/src/program/lwaftr/run_nohw/README b/src/program/lwaftr/run_nohw/README index 35cf6dafec..9a824eedf0 100644 --- a/src/program/lwaftr/run_nohw/README +++ b/src/program/lwaftr/run_nohw/README @@ -5,7 +5,7 @@ Usage: run-nohw --help --b4-if, -B NAME Name of the B4-side network interface. --inet-if, -I NAME Name of the Internet-side network interface. --verbose, -v Be verbose. Can be used multiple times. - --bench-file FILENAME + --bench-file, -b FILENAME The file or path name to which benchmark data is written. A simple filename or relative pathname will be based on the current directory. Default diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index 2a62e9dbdc..51ab101adf 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -33,7 +33,7 @@ local function parse_args(args) I = function (arg) inet_if = arg end; - ["bench-file"] = function (arg) + b = function (arg) bench_file = arg end; h = function (arg) @@ -44,7 +44,7 @@ local function parse_args(args) lib.dogetopt(args, handlers, "b:c:B:I:vh", { help = "h", conf = "c", verbose = "v", ["b4-if"] = "B", ["inet-if"] = "I", - ["bench-file"] = 0, + ["bench-file"] = "b", }) check(conf_file, "no configuration specified (--conf/-c)") check(b4_if, "no B4-side interface specified (--b4-if/-B)") @@ -73,7 +73,7 @@ function run(parameters) config.link(c, "aftr.v6 -> b4if.rx") if verbosity >= 1 then - local csv = CSVStatsTimer.new(bench_file) + local csv = CSVStatsTimer:new(bench_file) csv:add_app("inet", {"tx", "rx"}, { tx = "IPv4 TX", rx = "IPv4 RX" }) csv:add_app("tob4", {"tx", "rx"}, { tx = "IPv6 TX", rx = "IPv6 RX" }) csv:activate() diff --git a/src/program/lwaftr/transient/transient.lua b/src/program/lwaftr/transient/transient.lua index 639a0db821..00110a3f64 100644 --- a/src/program/lwaftr/transient/transient.lua +++ b/src/program/lwaftr/transient/transient.lua @@ -126,7 +126,7 @@ function run(args) rate_adjuster() timer.activate(timer.new("adjust_rate", rate_adjuster, opts.duration * 1e9, 'repeating')) - local csv = csv_stats.CSVStatsTimer.new(opts.bench_file) + local csv = csv_stats.CSVStatsTimer:new(opts.bench_file) for _,stream in ipairs(streams) do csv:add_app(stream.nic_id, { 'rx', 'tx' }, { rx=stream.name..' TX', tx=stream.name..' RX' }) From 0ff7a430bf4b87c62b672e5de29e943caceb4fd9 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 10:11:46 +0100 Subject: [PATCH 075/631] Yang data parsing/grammar cleanups Generate data grammar from schema, then generate parser from grammar. --- src/lib/yang/data.lua | 136 ++++++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 44 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 8f9fa77d99..efa6adbbe7 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -6,6 +6,42 @@ local ffi = require("ffi") local parser = require("lib.yang.parser") local schema = require("lib.yang.schema2") +function data_grammar_from_schema(schema) + local handlers = {} + local function visit(node) + local handler = handlers[node.kind] + if handler then return handler(node) end + return {} + end + local function visit_body(node) + local ret = {} + for id,node in pairs(node.body) do + for keyword,node in pairs(visit(node)) do + assert(not ret[keyword], 'duplicate identifier: '..keyword) + ret[keyword] = node + end + end + return ret + end + function handlers.container(node) + if not node.presence then return visit_body(node) end + return {[node.id]={type='struct', members=visit_body(node)}} + end + handlers['leaf-list'] = function(node) + return {[node.id]={type='array', element_type=node.type}} + end + function handlers.list(node) + local keys = {} + for key in node.key:split(' +') do table.insert(keys, key) end + return {[node.id]={type='table', keys=keys, members=visit_body(node)}} + end + function handlers.leaf(node) + return {[node.id]={type='scalar', argument_type=node.type, + default=node.default, mandatory=node.mandatory}} + end + return {type="struct", members=visit_body(schema)} +end + ffi.cdef([[ unsigned long long strtoull (const char *nptr, const char **endptr, int base); ]]) @@ -112,23 +148,38 @@ local function value_parser(typ) end end -local function struct_parser(members, k) +local function assert_scalar(node, keyword, opts) + assert(node.argument or (opts and opts.allow_empty_argument), + 'missing argument for "'..keyword..'"') + assert(not node.statements, 'unexpected sub-parameters for "'..keyword..'"') +end + +local function assert_compound(node, keyword) + assert(not node.argument, 'argument unexpected for "'..keyword..'"') + assert(node.statements, + 'missing brace-delimited sub-parameters for "'..keyword..'"') +end + +local function assert_not_duplicate(out, keyword) + assert(not out, 'duplicate parameter: '..keyword) +end + +local function struct_parser(keyword, members) local function init() return nil end local function parse1(node) - assert(not node.argument, 'argument unexpected for struct type: '..k) - assert(node.statements, 'missing statements for struct type: '..k) + assert_compound(node, keyword) local ret = {} for k,sub in pairs(members) do ret[k] = sub.init() end for _,node in ipairs(node.statements) do local sub = assert(members[node.keyword], - 'unrecognized keyword: '..node.keyword) + 'unrecognized parameter: '..node.keyword) ret[node.keyword] = sub.parse(node, ret[node.keyword]) end for k,sub in pairs(members) do ret[k] = sub.finish(ret[k]) end return ret end local function parse(node, out) - if out then error('duplicate struct: '..k) end + assert_not_duplicate(out, keyword) return parse1(node) end local function finish(out) @@ -138,12 +189,11 @@ local function struct_parser(members, k) return {init=init, parse=parse, finish=finish} end -local function array_parser(typ, k) +local function array_parser(keyword, element_type) local function init() return {} end - local parsev = value_parser(typ) + local parsev = value_parser(element_type) local function parse1(node) - assert(node.argument, 'argument expected for array type: '..k) - assert(not node.statements, 'unexpected statements for array type: '..k) + assert_scalar(node, keyword) return parsev(node.argument, k) end local function parse(node, out) @@ -157,15 +207,15 @@ local function array_parser(typ, k) return {init=init, parse=parse, finish=finish} end -local function scalar_parser(typ, default, mandatory, k) +local function scalar_parser(keyword, argument_type, default, mandatory) local function init() return nil end - local parsev = value_parser(typ) + local parsev = value_parser(argument_type) local function parse1(node) - assert(not node.statements, 'unexpected statements for scalar type: '..k) - return parsev(node.argument, k) + assert_scalar(node, keyword, {allow_empty_argument=true}) + return parsev(node.argument, keyword) end local function parse(node, out) - assert(not out, 'duplicate scalar config value: '..k) + assert_not_duplicate(out, keyword) return parse1(node) end local function finish(out) @@ -176,20 +226,21 @@ local function scalar_parser(typ, default, mandatory, k) return {init=init, parse=parse, finish=finish} end -local function table_parser(keystr, members, k) +local function table_parser(keyword, keys, members) -- This is a temporary lookup until we get the various Table kinds -- working. local function lookup(out, k) for _,v in ipairs(out) do - if v[keystr] == k then return v end + if #keys == 1 then + if v[keys[1]] == k then return v end + end end error('not found: '..k) end local function init() return {lookup=lookup} end - local parser = struct_parser(members, k) + local parser = struct_parser(keyword, members) local function parse1(node) - assert(not node.argument, 'argument unexpected for table type: '..k) - assert(node.statements, 'expected statements for table type: '..k) + assert_compound(node, keyword) return parser.finish(parser.parse(node, parser.init())) end local function parse(node, out) @@ -205,42 +256,39 @@ local function table_parser(keystr, members, k) end function data_parser_from_schema(schema) + return data_parser_from_grammar(data_grammar_from_schema(schema)) +end + +function data_parser_from_grammar(production) local handlers = {} - local function visit(node) - local handler = handlers[node.kind] - if handler then return handler(node) end - return {} + local function visit1(keyword, production) + return assert(handlers[production.type])(keyword, production) end - local function visit_body(node) + local function visitn(productions) local ret = {} - for id,node in pairs(node.body) do - for keyword,node in pairs(visit(node)) do - assert(not ret[keyword], 'duplicate identifier: '..keyword) - ret[keyword] = node - end + for keyword,production in pairs(productions) do + ret[keyword] = visit1(keyword, production) end return ret end - function handlers.module(node) + function handlers.struct(keyword, production) + return struct_parser(keyword, visitn(production.members)) end - function handlers.container(node) - if not node.presence then return visit_body(node) end - return {[node.id]=struct_parser(visit_body(node), node.id)} - end - handlers['leaf-list'] = function(node) - return {[node.id]=array_parser(node.type, node.id)} + function handlers.array(keyword, production) + return array_parser(keyword, production.element_type) end - function handlers.list(node) - return {[node.id]=table_parser(node.key, visit_body(node), node.id)} + function handlers.table(keyword, production) + return table_parser(keyword, production.keys, visitn(production.members)) end - function handlers.leaf(node) - return {[node.id]=scalar_parser(node.type, node.default, node.mandatory, -node.id)} + function handlers.scalar(keyword, production) + return scalar_parser(keyword, production.argument_type, + production.default, production.mandatory) end - local parser = struct_parser(visit_body(schema), '(top level)') - return function(stmtlist) - return parser.finish(parser.parse({statements=stmtlist}, parser.init())) + local parser = visit1('(top level)', production) + return function(statement_list) + local node = {statements=statement_list} + return parser.finish(parser.parse(node, parser.init())) end end From e95af47d67dc1a8f00fec8a62e2321b7c345245e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 13:50:18 +0100 Subject: [PATCH 076/631] Add yang README.md --- src/lib/yang/README.md | 162 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 src/lib/yang/README.md diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md new file mode 100644 index 0000000000..66a8b28181 --- /dev/null +++ b/src/lib/yang/README.md @@ -0,0 +1,162 @@ +### Snabb program configuration with YANG (`lib.yang`) + +YANG is a data modelling language designed for use in networking +equipment, standardized as [RFC +6020](https://tools.ietf.org/html/rfc6020). The `lib.yang` modules +provide YANG facilities to Snabb applications, allowing operators to +understand how to work with a Snabb data plane and also providing +convenient configuration facilities for data-plane authors. + +#### Overview + +Everything in YANG starts with a *schema*: a specification of the data +model of a device. For example, consider a simple Snabb router that +receives IPv4 traffic and sends it out one of 12 ports. We might +model it like this: + +```yang +module snabb-simple-router { + namespace snabb:simple-router; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + leaf active { type boolean; default true; } + + container routes { + presence true; + list route { + key addr; + leaf addr { type inet:ipv4-address; mandatory true; } + leaf port { type uint8 { range 0..11; }; mandatory true; } + } + } +} +``` + +Given this schema, `lib.yang` can automatically derive a configuration +file format for this Snabb program and create a parser that applies +the validation constraints from the schema. The result is a simple +plain-old-data Lua object that the data-plane can use directly. + +Additionally there is support for efficient binary compilation of +configurations. The problem is that even in this simple router, the +routing table can grow quite large. While particular applications can +sometimes incrementally update their configurations without completely +reloading the configuration from the start, in general reloading is +almost always a possibility, and you want to avoid packet loss during +the time that the millions of routing table entries are loaded and +validated. + +For that reason the `lib.yang` code also defines a mapping that, given +a YANG schema, can compile any configuration for that schema into a +pre-validated binary file that the data-plane can just load up +directly. Additionally for `list` nodes that map between keys and +values, the `lib.yang` facilities can compile that map into an +efficient [`ctable`](../README.ctable.md), letting the data-plane use +the configuration as-is. + +The schema given above can be loaded from a string using `load_schema` +from the `lib.yang.schema` module, from a file via `load_schema_file`, +or by name using `load_schema_by_name`. This last interface allows +one to compile a YANG schema into the Snabb binary directly; if we +name the above file `snabb-simple-router.yang` and place it in the +`src/lib/yang` directory, then +`load_schema_by_name('snabb-simple-router')` will find it +appropriately. Indeed, this is how the `ietf-inet-types` import in +the above example was resolved. + +#### Configuration syntax + +Consider again the example `snabb-simple-router` schema. To configure +a router, we need to provide a configuration in a way that the +application can understand. In Snabb, we derive this configuration +syntax from the schema, in the following way: + +- A `module`'s configuration is composed of the configurations of all + data nodes (`container`, `leaf-list`, `list`, and `leaf`) nodes + inside it. + +- A `leaf`'s configuration is like `keyword value;`, where the keyword + is the name of the leaf, and the value is in the right syntax for + the leaf's type. (More on value types below.) + +- A `container`'s configuration can be one of two ways. Firstly, if + its `presence` attribute is `true`, then the container's + configuration is the container's keyword followed by the + configuration of its data node children, like `keyword { + configuration... }`. Otherwise if its `presence` is `false`, then a + container's configuration is just its data node children's + configuration, in any order. + +- A `leaf-list`'s configuration is a sequence of 0 or more instances + of `keyword value;`, as in `leaf`. + +- A `list`'s configuration is a sequence of 0 or more instances of the + form `keyword { configuration... }`, again where `keyword` is the + list name and `configuration...` indicates the configuration of + child data nodes. + +Concretely, for the example configuration above, the above algorithm +derives a configuration format of the following form: + +``` +(active true|false;)? +(routes { + (route { addr ipv4-address; port uint8; })* +})? +``` + +In this grammar syntax, `(foo)?` indicates either 0 or 1 instances of +`foo`, `(foo)*` is similar bit indicating 0 or more instances, and `|` +expresses alternation. + +An example configuration might be: + +``` +active true; +routes { + route { addr 1.2.3.4; port 1; } + route { addr 2.3.4.5; port 10; } + route { addr 3.4.5.6; port 2; } +} +``` + +Except in special cases as described in RFC 6020, order is +insignificant. You could have `active false;` at the end, for +example, and `route { addr 1.2.3.4; port 1; }` is the same as `route { +port 1; addr 1.2.3.4; }`. Note that if `presence` is false (the +default), the grammar is the same except there's no outer `routes { }` +wrapper; the `route` statements would be at the same level as +`active`. + +The surface syntax of our configuration format is the same as for YANG +schemas; `"1.2.3.4"` is the same as `1.2.3.4`. Snabb follows the XML +mapping guidelines of how to represent data described by a YANG +schema, except that it uses YANG syntax instead of XML syntax. We +could generate XML instead, but we want to avoid bringing in the +complexities of XML parsing to Snabb. We also think that the result +is a syntax that is pleasant and approachable to write by hand; we +want to make sure that everyone can use the same configuration format, +regardless of whether they are configuring Snabb via an external +daemon like `sysrepo` or whether they write configuration files by +hand. + +#### Compiled configurations + +#### Querying and updating configurations + +#### State data + +#### Function reference + +All of these functions are on modules in the `lib.yang` path. For +example, to have access to: + +— Function **schema.load_schema_by_name** *name* + +Then do `local schema = require('lib.yang.schema')`. + +— Function **schema.load_schema_by_name** *name* + +Load up schema by name. [TODO write more here.] From 36f4ab01fd9af7fd7eda9a709623be4e1ef91c97 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 14:13:40 +0100 Subject: [PATCH 077/631] Simplify lib.yang.data interfaces Add load_data_for_schema et al interfaces. --- src/lib/yang/data.lua | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index efa6adbbe7..09172c394d 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -3,7 +3,7 @@ module(..., package.seeall) local ffi = require("ffi") -local parser = require("lib.yang.parser") +local parse_string = require("lib.yang.parser").parse_string local schema = require("lib.yang.schema2") function data_grammar_from_schema(schema) @@ -286,14 +286,23 @@ function data_parser_from_grammar(production) end local parser = visit1('(top level)', production) - return function(statement_list) - local node = {statements=statement_list} + return function(str, filename) + local node = {statements=parse_string(str, filename)} return parser.finish(parser.parse(node, parser.init())) end end +function load_data_for_schema(schema, str, filename) + return data_parser_from_schema(schema)(str, filename) +end + +function load_data_for_schema_by_name(schema_name, str, filename) + local schema = schema.load_schema_by_name(schema_name) + return load_data_for_schema(schema, str, filename) +end + function selftest() - local test_schema = [[module fruit { + local test_schema = schema.load_schema([[module fruit { namespace "urn:testing:fruit"; prefix "fruit"; grouping fruit { @@ -313,18 +322,16 @@ function selftest() leaf description { type string; } list contents { uses fruit; key name; } } - }]] + }]]) - local schema = schema.load_schema(test_schema) - local parse = data_parser_from_schema(schema) - local data = parse(parser.parse_string([[ + local data = load_data_for_schema(test_schema, [[ fruit-bowl { description 'ohai'; contents { name foo; score 7; } contents { name bar; score 8; } contents { name baz; score 9; tree-grown true; } } - ]])) + ]]) assert(data['fruit-bowl'].description == 'ohai') assert(data['fruit-bowl'].contents:lookup('foo').name == 'foo') assert(data['fruit-bowl'].contents:lookup('foo').score == 7) From 6bf082e4ef7ee3550f6ea28564aaa098d8b1c12b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 14:24:46 +0100 Subject: [PATCH 078/631] Rename lib.yang.schema2 to lib.yang.schema * src/lib/yang/schema.lua: Rename from schema2.lua. * src/lib/yang/yang.lua: Just re-export things from schema and data modules. Document via README.md. --- src/lib/yang/data.lua | 2 +- src/lib/yang/schema.lua | 1134 ++++++++++++++++++++++++++------------ src/lib/yang/schema2.lua | 927 ------------------------------- src/lib/yang/yang.lua | 412 +------------- 4 files changed, 790 insertions(+), 1685 deletions(-) delete mode 100644 src/lib/yang/schema2.lua diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 09172c394d..7eb8144709 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -4,7 +4,7 @@ module(..., package.seeall) local ffi = require("ffi") local parse_string = require("lib.yang.parser").parse_string -local schema = require("lib.yang.schema2") +local schema = require("lib.yang.schema") function data_grammar_from_schema(schema) local handlers = {} diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 0b5bc13cae..fc5973e5a4 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -6,374 +6,808 @@ -- Since YANG statements are encapsulated in modules at the highest level one -- should take their pre-parsed YANG document containing the module and load it -- into the Module table. --- --- This relies on the "Base" table which can be found in the yang.lua file. module(..., package.seeall) - -local helpers = require("lib.yang.helpers") -local h = require("syscall.helpers") - -local Leaf = {} -local List = {} -local Feature = {} -local Grouping = {} -local Container = {} -local Revision = {} -local List = {} -Module = {} - -function Leaf.new(base, path, src) - local self = setmetatable({}, {__index=Leaf, path=path}) - - -- Parse the schema to find the metadata - self:validate_schema(src) - base:add_cache(path, self) - - self.type = src.type[1].argument - if src.type[1].statements then - local typeinfo = src.type[1].statements - if typeinfo.range then - local range = h.split("%.%.", typeinfo.range[1].argument) - self.range = {tonumber(range[1]), tonumber(range[2])} - elseif typeinfo.enum then - self.enums = {} - for _, v in pairs(typeinfo.enum) do - self.enums[v.argument] = v.argument - end - elseif self.type == "union" and typeinfo.type then - self.types = {} - for _, v in pairs(typeinfo.type) do - self.types[#self.types + 1] = v.argument - end - end - end - if src.description then - self.description = src.description[1].argument - end - - if src.default then - self.default = src.default[1].argument - end - - if src["if-feature"] then - self["if-feature"] = {} - for _, f in pairs(src["if-feature"]) do - table.insert(self["if-feature"], f.argument) - end - end - - if src.mandatory then - self.mandatory = src.mandatory[1].argument == "true" - end - - -- Add validators if we need to. - if self.mandatory then - if not self.validation then self.validation = {} end - table.insert(self.validation, function (v) - if not v then - self:error("Value is mandatory") - end - end) - end - if self.range then - if not self.validation then self.validation = {} end - self.validation[#self.validation + 1] = function(v) - v = tonumber(v) - if v < self.range[1] or v > self.range[2] then - self:error("Value '%s' is out of range", path, v) - end - end - end - if self.enums then - if not self.validation then self.validation = {} end - self.validation[#self.validation + 1] = function (v) - if v and not self.enums[v] then - self:error("Value '%s' is not one of the Enum values", v) - end - end +local parser = require("lib.yang.parser") + +local ffi = require("ffi") +ffi.cdef("long long atoll(const char *nptr);") +local function tointeger(str) + local i = ffi.C.atoll(str) + if tostring(i) == str.."LL" then + if i == tonumber(i) then return tonumber(i) else return i end end - return self end -function Leaf:get_type() return "leaf" end - -function Leaf:error(msg, ...) - local path = getmetatable(self).path - error(("%s: %s"):format(path, msg:format(...))) +local function error_with_path(path, msg, ...) + error(string.format("%s: "..msg, path, ...)) end - -function Leaf:validate_schema(schema) - local cardinality = {config={0,1}, default={0,1}, description={0,1}, - mandatory={0,1}, reference={0,1}, status={0,1}, - type={1,1}, units={0,1}, when={0,1}} - helpers.cardinality("leaf", getmetatable(self).path, cardinality, schema) +local function assert_with_path(expr, path, msg, ...) + if not expr then error_with_path(path, msg, ...) end + return expr end --- Yang feature -function Feature.new(base, path, src) - local self = setmetatable({}, {__index=Feature, path=path}) - - self:validate_schema(src) - base:add_cache(path, self) - - if src.description then - self.description = src.description[1].argument - end - - if src.reference then - self.reference = src.reference[1].argument +-- (kind -> (function(Node) -> value)) +local initializers = {} +local function declare_initializer(init, ...) + for _, keyword in ipairs({...}) do initializers[keyword] = init end +end + +local Node = {} +local function parse_node(src, parent_path, order) + local ret = {} + ret.kind = assert(src.keyword, 'missing keyword') + if parent_path then + ret.path = parent_path..'.'..ret.kind + else + ret.path = ret.kind end + ret.order = order + ret.argument_string = src.argument + ret.children = parse_children(src, ret.path) + ret = setmetatable(ret, {__index=Node}) + local initialize = initializers[ret.kind] + if initialize then initialize(ret) end + return ret +end - if src.status then - self.status = src.reference[1].argument +function parse_children(src, parent_path) + local ret = {} + for i, statement in ipairs(src.statements or {}) do + local child = parse_node(statement, parent_path, i) + if not ret[child.kind] then ret[child.kind] = {} end + table.insert(ret[child.kind], child) end - - return self + return ret end -function Feature:get_type() return "feature" end - -function Feature:validate_schema(src) - local cardinality = {description={0,1}, status={0,1}, refernece={0,1}} - helpers.cardinality("feature", getmetatable(self).path, cardinality, src) +local function require_argument(node) + return assert_with_path(node.argument_string, node.path, + 'missing argument') end --- Yang list -function List.new(base, path, src) - local ret = {leaves={}, containers={}} - local self = setmetatable(ret, {__index=List, path=path}) - - self:validate_schema(src) - base:add_cache(path, self) - - if src.key then self.key = src.key[1].argument end - if src.uses then self.uses = src.uses[1].argument end - if src.leaf then - for _, leaf in pairs(src.leaf) do - local path = path.."."..leaf.argument - self.leaves[leaf.argument] = Leaf.new(base, path, leaf.statements) +local function parse_range(node, range) + local function parse_part(part) + local l, r = part:match("^%s*([^%.]*)%s*%.%.%s*([^%s]*)%s*$") + assert_with_path(l, node.path, 'bad range component: %s', part) + if l ~= 'min' then + l = assert_with_path(tointeger(l), node.path, "bad integer: %s", l) end - end - if src.container then - for _, container in pairs(src.container) do - local path = path.."."..container.argument - self.containers[container.argument] = Container.new(base, path, container.statements) + if r ~= 'max' then + r = assert_with_path(tointeger(r), node.path, "bad integer: %s", r) end + return { l, r } end - - return self -end -function List:get_type() return "list" end - -function List:validate_schema(src) - local cardinality = {config={0,1}, description={0,1}, key={0,1}, - reference={0,1}, status={0,1}, when={0,1}} - cardinality["max-elements"] = {0,1} - cardinality["min-elements"] = {0,1} - cardinality["ordered-by"] = {0,1} - helpers.cardinality("list", getmetatable(self).path, cardinality, src) + local parts = range:split("|") + local res = {'or'} + for part in range:split("|") do table.insert(res, parse_part(part)) end + if #res == 1 then error_with_path(node.path, "empty range", range) + elseif #res == 2 then return res[2] + else return res end end --- Yang group -function Grouping.new(base, path, src) - local ret = {leaves={}} - local self = setmetatable(ret, {__index = Grouping, path=path}) - - self:validate_schema(src) - base:add_cache(path, self) - - if src.description then - self.description = src.description[1].argument - end - - if src.list then - for _, list in pairs(src.list) do - local path = path.."."..list.argument - self[list.argument] = List.new(base, path, list.statements) +local function collect_children(node, kinds) + if type(kinds) == 'string' then return collect_children(node, {kinds}) end + local ret = {} + for _, kind in ipairs(kinds) do + if node.children[kind] then + for _, child in pairs(node.children[kind]) do + table.insert(ret, child) + end end end + return ret +end - if src.leaf then - for _, leaf in pairs(src.leaf) do - local path = path.."."..leaf.argument - self.leaves[leaf.argument] = Leaf.new(base, path, leaf.statements) - end +local function collect_children_by_prop(node, kinds, prop) + local ret = {} + for _, child in ipairs(collect_children(node, kinds)) do + assert_with_path(child[prop], node.path, + 'child of kind %s missing prop %s', child.kind, prop) + assert_with_path(not ret[child[prop]], node.path, + 'duplicate %s: %s', prop, child[prop]) + ret[child[prop]] = child end - - return self + return ret end -function Grouping:get_type() return "grouping" end -function Grouping:validate_schema(src) - local cardinality = {description={0,1}, reference={0,1}, status={0,1}} - helpers.cardinality("grouping", getmetatable(self).path, cardinality, src) +local function collect_children_by_id(node, kinds) + return collect_children_by_prop(node, kinds, 'id') end -function Container.new(base, path, src) - local ret = {leaves={}, containers={}, lists={}} - local self = setmetatable(ret, {__index=Container, path=path}) +local function collect_body_children(node) + return collect_children_by_id( + node, + {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml'}) +end - self:validate_schema(src) - base:add_cache(path, self) +local function at_least_one(tab) + for k, v in pairs(tab) do return true end + return false +end - if src.description then - self.description = src.description[1].argument +local function collect_body_children_at_least_1(node) + local ret = collect_body_children(node) + if not at_least_one(ret) then + error_with_path(node.path, "missing data statements") end + return ret +end - -- Leaf statements - if src.leaf then - for _, leaf in pairs(src.leaf) do - self.leaves[leaf.argument] = Leaf.new( - base, - path.."."..leaf.argument, - leaf.statements - ) - end +local function collect_data_or_case_children_at_least_1(node) + local ret = collect_children_by_id( + node, + {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', + 'anyxml', 'case'}) + if not at_least_one(ret) then + error_with_path(node.path, "missing data statements") end + return ret +end - -- Include other containers - if src.container then - for _, container in pairs(src.container) do - self.containers[container.argument] = Container.new( - base, - path.."."..container.argument, - container.statements - ) - end +local function collect_child_properties(node, kind, field) + local ret = {} + for _, child in ipairs(collect_children(node, kind)) do + table.insert(ret, child[field]) end + return ret +end - if src.list then - for _, list in pairs(src.list) do - self.lists[list.argument] = List.new( - base, - path.."."..list.argument, - list.statements - ) - end +local function maybe_child(node, kind) + local children = collect_children(node, kind) + if #children > 1 then + error_with_path(node.path, 'expected at most one child of type %s', kind) end + return children[1] +end - if src.uses then - self.uses = src.uses[1].argument - end +local function maybe_child_property(node, kind, prop) + local child = maybe_child(node, kind) + if child then return child[prop] end +end - return self +local function require_child(node, kind) + local child = maybe_child(node, kind) + if child then return child end + error_with_path(node.path, 'missing child of type %s', kind) end -function Container:get_type() return "container" end -function Container:validate_schema(src) - local cardinality = {config={0,1}, description={0,1}, presense={0,1}, - reference={0,1}, status={0,1}, when={0,1}} - helpers.cardinality("container", getmetatable(self).path, cardinality, src) +local function require_child_property(node, kind, prop) + return require_child(node, kind)[prop] end --- Yang Revision -function Revision.new(base, path, src) - local self = setmetatable({}, {__index=Revision, path=path}) +-- Simple statement kinds with string, natural, or boolean values all +-- just initialize by parsing their argument and storing it as the +-- "value" property in the schema node. +local function init_string(node) + node.value = require_argument(node) +end +local function init_natural(node) + local arg = require_argument(node) + local as_num = tonumber(arg) + assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, + node.path, 'not a natural number: %s', arg) + node.value = as_num +end +local function init_boolean(node) + local arg = require_argument(node) + if arg == 'true' then node.value = true + elseif arg == 'false' then node.value = false + else error_with_path(node.path, 'not a valid boolean: %s', arg) end +end - self:validate_schema(src) - base:add_cache(path, self) +-- For all other statement kinds, we have custom initializers that +-- parse out relevant sub-components and store them as named +-- properties on the schema node. +local function init_anyxml(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_argument(node) + node.id = require_argument(node) + node.yin_element = maybe_child_property(node, 'yin-element', 'value') +end +local function init_augment(node) + node.node_id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.body = collect_data_or_case_children_at_least_1(node) +end +local function init_belongs_to(node) + node.id = require_argument(node) + node.prefix = require_child(node, 'prefix').value +end +local function init_case(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.body = collect_body_children(node) +end +local function init_choice(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.default = maybe_child_property(node, 'default', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.body = collect_children_by_id( + node, + {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) +end +local function init_container(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.presence = maybe_child_property(node, 'presence', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.body = collect_body_children(node) +end +local function init_extension(node) + node.id = require_argument(node) + node.argument = maybe_child_property(node, 'argument', 'id') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_feature(node) + node.id = require_argument(node) + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_grouping(node) + node.id = require_argument(node) + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.body = collect_body_children(node) +end +local function init_identity(node) + node.id = require_argument(node) + node.base = maybe_child_property(node, 'base', 'id') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_import(node) + node.id = require_argument(node) + node.prefix = require_child_property(node, 'prefix', 'value') + node.revision_date = maybe_child_property(node, 'revision-date', 'value') +end +local function init_include(node) + node.id = require_argument(node) + node.revision_date = maybe_child_property(node, 'revision-date', 'value') +end +local function init_input(node) + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.body = collect_body_children_at_least_1(node) +end +local function init_leaf(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.type = require_child(node, 'type') + node.units = maybe_child_property(node, 'units', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.default = maybe_child_property(node, 'default', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_leaf_list(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.type = require_child(node, 'type') + node.units = maybe_child_property(node, 'units', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.min_elements = maybe_child_property(node, 'min-elements', 'value') + node.max_elements = maybe_child_property(node, 'max-elements', 'value') + node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_length(node) + -- TODO: parse length arg str + node.value = require_argument(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_list(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.must = collect_child_properties(node, 'must', 'value') + node.key = maybe_child_property(node, 'key', 'value') + node.unique = collect_child_properties(node, 'unique', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.min_elements = maybe_child_property(node, 'min-elements', 'value') + node.max_elements = maybe_child_property(node, 'max-elements', 'value') + node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.body = collect_body_children_at_least_1(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_module(node) + node.id = require_argument(node) + node.yang_version = maybe_child_property(node, 'yang-version', 'value') + node.namespace = require_child_property(node, 'namespace', 'value') + node.prefix = require_child_property(node, 'prefix', 'value') + node.imports = collect_children_by_prop(node, 'import', 'prefix') + node.includes = collect_children_by_id(node, 'include') + node.organization = maybe_child_property(node, 'organization', 'value') + node.contact = maybe_child_property(node, 'contact', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.revisions = collect_children(node, 'revision') + node.augments = collect_children(node, 'augment') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.features = collect_children_by_id(node, 'feature') + node.extensions = collect_children_by_id(node, 'extension') + node.identities = collect_children_by_id(node, 'identity') + node.rpcs = collect_children_by_id(node, 'rpc') + node.notifications = collect_children_by_id(node, 'notification') + node.deviations = collect_children_by_id(node, 'deviation') + node.body = collect_body_children(node) +end +local function init_namespace(node) + -- TODO: parse uri? + node.value = require_argument(node) +end +local function init_notification(node) + node.id = require_argument(node) + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.body = collect_body_children(node) +end +local function init_output(node) + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.body = collect_body_children_at_least_1(node) +end +local function init_path(node) + -- TODO: parse path string + node.value = require_argument(node) +end +local function init_pattern(node) + node.value = require_argument(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_range(node) + node.value = parse_range(node, require_argument(node)) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_refine(node) + node.node_id = require_argument(node) + -- All subnode kinds. + node.must = collect_child_properties(node, 'must', 'value') + node.config = maybe_child_property(node, 'config', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + -- Containers. + node.presence = maybe_child_property(node, 'presence', 'value') + -- Leaves, choice, and (for mandatory) anyxml. + node.default = maybe_child_property(node, 'default', 'value') + node.mandatory = maybe_child_property(node, 'mandatory', 'value') + -- Leaf lists and lists. + node.min_elements = maybe_child_property(node, 'min-elements', 'value') + node.max_elements = maybe_child_property(node, 'max-elements', 'value') +end +local function init_revision(node) + -- TODO: parse date + node.value = require_argument(node) + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_rpc(node) + node.id = require_argument(node) + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.input = maybe_child(node, 'input') + node.output = maybe_child(node, 'output') +end +local function init_type(node) + node.id = require_argument(node) + node.range = maybe_child(node, 'range') + node.fraction_digits = maybe_child_property(node, 'fraction-digits', 'value') + node.length = maybe_child_property(node, 'length', 'value') + node.patterns = collect_children(node, 'pattern') + node.enums = collect_children(node, 'enum') + -- !!! path + node.leafref = maybe_child_property(node, 'path', 'value') + node.require_instances = collect_children(node, 'require-instance') + node.identityref = maybe_child_property(node, 'base', 'value') + node.union = collect_children(node, 'type') + node.bits = collect_children(node, 'bit') +end +local function init_submodule(node) + node.id = require_argument(node) + node.yang_version = maybe_child_property(node, 'yang-version', 'value') + node.belongs_to = require_child(node, 'belongs-to') + node.imports = collect_children_by_id(node, 'import') + node.includes = collect_children_by_id(node, 'include') + node.organization = maybe_child_property(node, 'organization', 'value') + node.contact = maybe_child_property(node, 'contact', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.revisions = collect_children(node, 'revision') + node.augments = collect_children(node, 'augment') + node.typedefs = collect_children_by_id(node, 'typedef') + node.groupings = collect_children_by_id(node, 'grouping') + node.features = collect_children_by_id(node, 'feature') + node.extensions = collect_children_by_id(node, 'extension') + node.identities = collect_children_by_id(node, 'identity') + node.rpcs = collect_children_by_id(node, 'rpc') + node.notifications = collect_children_by_id(node, 'notification') + node.deviations = collect_children_by_id(node, 'deviation') + node.body = collect_body_children(node) +end +local function init_typedef(node) + node.id = require_argument(node) + node.type = require_child(node, 'type') + node.units = maybe_child_property(node, 'units', 'value') + node.default = maybe_child_property(node, 'default', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') +end +local function init_uses(node) + node.id = require_argument(node) + node.when = maybe_child_property(node, 'when', 'value') + node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.status = maybe_child_property(node, 'status', 'value') + node.description = maybe_child_property(node, 'description', 'value') + node.reference = maybe_child_property(node, 'reference', 'value') + node.typedefs = collect_children_by_id(node, 'typedef') + node.refines = collect_children(node, 'refine') + node.augments = collect_children(node, 'augment') +end +local function init_value(node) + local arg = require_argument(node) + local as_num = tonumber(arg) + assert_with_path(as_num and math.floor(as_num) == as_num, + node.path, 'not an integer: %s', arg) + node.value = as_num +end - if src.description then - self.description = src.description[1].argument +declare_initializer( + init_string, 'prefix', 'organization', 'contact', 'description', + 'reference', 'units', 'revision-date', 'base','if-feature', + 'default', 'enum', 'bit', 'status', 'presence', 'ordered-by', 'must', + 'error-message', 'error-app-tag', 'max-value', 'key', 'unique', 'when', + 'deviation', 'deviate') +declare_initializer( + init_natural, 'yang-version', 'fraction-digits', 'position', + 'min-elements', 'max-elements') +declare_initializer( + init_boolean, 'config', 'mandatory', 'require-instance', 'yin-element') +declare_initializer(init_anyxml, 'anyxml') +declare_initializer(init_argument, 'argument') +declare_initializer(init_augment, 'augment') +declare_initializer(init_belongs_to, 'belongs-to') +declare_initializer(init_case, 'case') +declare_initializer(init_choice, 'choice') +declare_initializer(init_container, 'container') +declare_initializer(init_extension, 'extension') +declare_initializer(init_feature, 'feature') +declare_initializer(init_grouping, 'grouping') +declare_initializer(init_identity, 'identity') +declare_initializer(init_import, 'import') +declare_initializer(init_include, 'include') +declare_initializer(init_input, 'input') +declare_initializer(init_leaf, 'leaf') +declare_initializer(init_leaf_list, 'leaf-list') +declare_initializer(init_length, 'length') +declare_initializer(init_list, 'list') +declare_initializer(init_module, 'module') +declare_initializer(init_namespace, 'namespace') +declare_initializer(init_notification, 'notification') +declare_initializer(init_output, 'output') +declare_initializer(init_path, 'path') +declare_initializer(init_pattern, 'pattern') +declare_initializer(init_range, 'range') +declare_initializer(init_refine, 'refine') +declare_initializer(init_revision, 'revision') +declare_initializer(init_rpc, 'rpc') +declare_initializer(init_submodule, 'submodule') +declare_initializer(init_type, 'type') +declare_initializer(init_typedef, 'typedef') +declare_initializer(init_uses, 'uses') +declare_initializer(init_value, 'value') + +local function schema_from_ast(ast) + local ret + local submodules = {} + for _,node in ipairs(ast) do + if node.keyword == 'module' then + assert(not ret, 'expected only one module form') + ret = parse_node(node) + elseif node.keyword == 'submodule' then + assert(not submodules[node.id], 'duplicate submodule name: '..node.id) + submodules[node.id] = parse_node(node) + else + error('expected only module and submodule statements, got: '..node.keyword) + end end + assert(ret, 'missing module form') + ret.submodules = submodules + return ret +end - if src.reference then - self.reference = src.reference[1].argument +-- Strip properties pertaining to original source representation. +local function strip(exp) + if type(exp) ~= 'table' then return exp end + local ret = {} + for k, v in pairs(exp) do + if k ~= 'children' and k ~= 'argument_string' and k ~= 'order' and k ~= 'path' then + ret[k] = strip(v) + end end - return self + return ret end -function Revision:get_type() return "revision" end -function Revision:validate_schema(src) - local cardinality = {description={0,1}, reference={0,1}} - helpers.cardinality("revision", getmetatable(self).path, cardinality, src) +local function set(...) + local ret = {} + for k, v in pairs({...}) do ret[v] = true end + return ret end --- Yang Module -function Module.new(base, name, src) - local ret = {body={}, name=name, modules={}, revisions={}, - features={}, groupings={}, containers={}} - local self = setmetatable(ret, {__index=Module, path=name}) - - -- TODO: remove me when proper loading support exists. - if not src then return self end - - -- Add self to path cache - base:add_cache(name, self) - - -- Validate the statements first. - self:validate_schema(src) - - -- Set the meta information about the module - self.namespace = src.namespace[1].argument - self.prefix = src.prefix[1].argument - - if src.organization then - self.organization = src.organization[1].argument +local primitive_types = set( + 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', + 'binary', 'bits', 'boolean', 'decimal64', 'empty', 'enumeration', + 'identityref', 'instance-identifier', 'leafref', 'string', 'union') + +-- Inline "grouping" into "uses". +-- Inline "submodule" into "include". +-- Inline "imports" into "module". +-- Inline "typedef" into "type". (TODO) +-- Resolve if-feature. +-- Warn on any "when", resolving them as being true. +-- Resolve all augment and refine nodes. (TODO) +function resolve(schema, features) + local function shallow_copy(node) + local out = {} + for k,v in pairs(node) do out[k] = v end + return out end - - if src.contact then - self.contact = src.contact[1].argument + local function pop_prop(node, prop) + local val = node[prop] + node[prop] = nil + return val end - - if src.description then - self.description = src.description[1].argument + local function lookup(env, prop, name) + if not env then error(prop..' not found: '..name) end + if not env[prop] or not env[prop][name] then + return lookup(env.env, prop, name) + end + return env[prop][name] end - - -- Now handle the imports, as other things may reference them. - if src.import then - for _, mod in pairs(src.import) do - self.modules[mod.argument] = Module.new(base, mod.argument) - - -- Ask the module to find and load itself. - self.modules[mod.argument]:load() + local function lookup_lazy(env, prop, name) + local val = lookup(env, prop, name) + if type(val) == 'table' then return val end + -- Force lazy expansion and memoize result. + return val() + end + local visit + local function visit_top_level(node, env, prop) + assert(not env[prop]) + env[prop] = {} + local p = lookup(env, 'prefix', '_') + for k,v in pairs(pop_prop(node, prop) or {}) do + env[prop][k] = visit(v, env) + env[prop][p..':'..k] = env[prop][k] end end - - -- Handle revisions - if src.revision then - for _, r in pairs(src.revision) do - local path = ret.name.."."..r.argument - self.revisions[r.argument] = Revision.new(base, path, r.statements) + local function visit_lazy(tab, env) + local ret = {} + local prefix = lookup(env, 'prefix', '_') + local function error_recursion() + end + for k,v in pairs(tab) do + -- FIXME: Only add prefix:k if at top level. + local state + local function lazy() + if state == 'visiting' then + error('mutually recursive definitions: '..k) + elseif state then + return state + else + state = 'visiting' + end + state = visit(v, env) + return state + end + ret[k] = lazy + ret[prefix..':'..k] = ret[k] end + return ret end - - -- Feature statements - if src.feature then - for _, f in pairs(src.feature) do - local path = ret.name.."."..f.argument - self.features[f.argument] = Feature.new(base, path, f.statements) + function visit_type(node, env) + node = shallow_copy(node) + local success, typedef = pcall(lookup, env, 'typedefs', node.id) + if success then + -- Could be that typedef is still lazy. We didn't use + -- lookup_lazy because we don't want the pcall to hide errors + -- from the lazy expansion. + if type(typedef) == 'function' then typedef = typedef() end + node.base_type = typedef + else + -- If the type name wasn't bound, it must be primitive. + assert(primitive_types[node.id], 'unknown type: '..node.id) + node.base_type = node.id end + return node end - - -- List statements - if src.grouping then - for _, g in pairs(src.grouping) do - local path = ret.name.."."..g.argument - self.groupings[g.argument] = Grouping.new(base, path, g.statements) + function visit(node, env) + node = shallow_copy(node) + env = {env=env} + if node.typedefs then + -- Populate node.typedefs as a table of thunks that will + -- lazily expand and memoize their result when called. This + -- is not only a performance optimization but also allows the + -- typedefs to be mutually visible. + env.typedefs = visit_lazy(pop_prop(node, 'typedefs'), env) + end + if node.groupings then + -- Likewise expand groupings at their point of definition. + env.groupings = visit_lazy(pop_prop(node, 'groupings'), env) + end + local when = pop_prop(node, 'when') + if when then + print('warning: assuming "when" condition to be true: '..when.value) end + if node.kind == 'module' or node.kind == 'submodule' then + visit_top_level(node, env, 'extensions') + visit_top_level(node, env, 'features') + visit_top_level(node, env, 'identities') + end + for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do + if not pcall(lookup, env, 'features', feature) then + return nil, env + end + end + if node.type then node.type = visit_type(node.type, env) end + for k,v in pairs(node.body or {}) do + if v.kind == 'uses' then + -- Inline "grouping" into "uses". + local grouping = lookup_lazy(env, 'groupings', v.id) + node.body[k] = nil + for k,v in pairs(grouping.body) do + assert(not node.body[k], 'duplicate identifier: '..k) + node.body[k] = v + end + -- TODO: Handle refine and augment statements. + else + node.body[k] = visit(v, env) + end + end + return node, env end - - -- Containers - if src.container then - for _, c in pairs(src.container) do - local path = ret.name.."."..c.argument - self.containers[c.argument] = Container.new(base, path, c.statements) + local function include(dst, src) + for k,v in pairs(src) do + assert(dst[k] == nil or dst[k] == v, 'incompatible definitions: '..k) + if not k:match(':') then dst[k] = v end end end - return self + local linked = {} + local function link(node, env) + if linked[node.id] then + assert(linked[node.id] ~= 'pending', 'recursive import of '..node.id) + local node, env = unpack(linked[node.id]) + return node, env + end + linked[node.id] = 'pending' + node = shallow_copy(node) + local module_env = {env=env, prefixes={}, extensions={}, features={}, + identities={}, typedefs={}, groupings={}} + local module_body = shallow_copy(node.body) + for k,v in pairs(pop_prop(node, 'includes')) do + local submodule = lookup(env, 'submodules', k) + assert(submodule.belongs_to.id == node.id) + submodule, submodule_env = link(submodule, env) + include(module_env.extensions, submodule_env.extensions) + include(module_env.features, submodule_env.features) + include(module_env.identities, submodule_env.identities) + include(module_env.typedefs, submodule_env.typedefs) + include(module_env.groupings, submodule_env.groupings) + include(module_body, submodule.body) + end + if node.prefix then + assert(node.kind == 'module', node.kind) + module_env.prefixes[node.prefix] = node.namespace + module_env.prefix = {_=node.prefix} + end + for k,v in pairs(pop_prop(node, 'imports')) do + assert(not module_env.prefixes[v.prefix], 'duplicate prefix') + -- CHECKME: Discarding body from import, just importing env. + -- Is this OK? + local schema, env = load_schema_by_name(v.id, v.revision_date) + local prefix = v.prefix + module_env.prefixes[prefix] = schema.namespace + for _,prop in ipairs({'extensions', 'features', 'identities', + 'typedefs', 'groupings'}) do + for k,v in pairs(env[prop]) do + if not k:match(':') then + module_env[prop][prefix..':'..k] = v + end + end + end + end + node, env = visit(node, module_env) + -- The typedefs, groupings, identities, and so on of this module + -- are externally visible for other modules that may import this + -- one; save them and their environment. + linked[node.id] = {node, env} + return node, env + end + schema = shallow_copy(schema) + return link(schema, {features=(features or {}), + submodules=pop_prop(schema, 'submodules')}) end -function Module:get_type() return "module" end -function Module:load() - -- TODO: Find the file and load it. +function parse_schema(src, filename) + return schema_from_ast(parser.parse_string(src, filename)) +end +function parse_schema_file(filename) + return schema_from_ast(parser.parse_file(filename)) end -function Module:validate_schema(src) - local cardinality = {contact={0,1}, description={0,1}, namespace={1,1}, - organization={0,1}, prefix={1,1}, reference={0,1}} - cardinality["yang-version"] = {0,1} - helpers.cardinality("module", getmetatable(self).path, cardinality, src) +function load_schema(src, filename) + return resolve(strip(parse_schema(src, filename))) +end +function load_schema_file(filename) + return resolve(strip(parse_schema_file(filename))) +end +function load_schema_by_name(name, revision) + -- FIXME: @ is not valid in a Lua module name. + -- if revision then name = name .. '@' .. revision end + name = name:gsub('-', '_') + return load_schema(require('lib.yang.'..name..'_yang'), name) end function selftest() @@ -406,7 +840,7 @@ function selftest() } grouping fruit { - description "Represets a piece of fruit"; + description "Represents a piece of fruit"; leaf name { type string; @@ -442,54 +876,52 @@ function selftest() } }]] - -- Convert the schema using the already tested parser. - local parser = require("lib.yang.parser") - local schema = parser.parse_string(test_schema, "schema selftest") - - -- Create a fake base, we're not testing this so avoidng using the real one. - local base = {add_cache = function() end} - - -- Convert the schema into a more usable form for us. - schema = helpers.extract_nodes(schema) - - -- Load the module - local mod = Module.new(base, schema.module[1].argument, - schema.module[1].statements) - - assert(mod.name == "fruit") - assert(mod.namespace == "urn:testing:fruit") - assert(mod.prefix == "fruit") - assert(mod.contact == "John Smith fake@person.tld") - assert(mod.organization == "Fruit Inc.") - assert(mod.description == "Module to test YANG schema lib") - - -- Check both modules exist. (Also need to check they've loaded) - assert(mod.modules["ietf-inet-types"]) - assert(mod.modules["ietf-yang-types"]) - - -- Check all revisions are accounted for. - assert(mod.revisions["2016-05-27"].description == "Revision 1") - assert(mod.revisions["2016-05-28"].description == "Revision 2") - - -- Check that the feature statements are there. - assert(mod.features["bowl"].description) - - -- Check the groupings - assert(mod.groupings["fruit"]) - assert(mod.groupings["fruit"].description) - assert(mod.groupings["fruit"].leaves["name"].type == "string") - assert(mod.groupings["fruit"].leaves["name"].mandatory == true) - assert(mod.groupings["fruit"].leaves["name"].description == "Name of fruit.") - assert(mod.groupings["fruit"].leaves["score"].type == "uint8") - assert(mod.groupings["fruit"].leaves["score"].mandatory == true) - assert(mod.groupings["fruit"].leaves["score"].range[1] == 0) - assert(mod.groupings["fruit"].leaves["score"].range[2] == 10) - - -- Check the containers description (NOT the leaf called "description") - assert(mod.containers["fruit-bowl"].description == "Represents a fruit bowl") - - -- Check the container has a leaf called "description" - local desc = mod.containers["fruit-bowl"].leaves.description - assert(desc.type == "string") - assert(desc.description == "About the bowl") + local schema, env = load_schema(test_schema) + assert(schema.id == "fruit") + assert(schema.namespace == "urn:testing:fruit") + assert(schema.prefix == "fruit") + assert(schema.contact == "John Smith fake@person.tld") + assert(schema.organization == "Fruit Inc.") + assert(schema.description == "Module to test YANG schema lib") + + -- Check all revisions are accounted for. + assert(schema.revisions[1].description == "Revision 1") + assert(schema.revisions[1].value == "2016-05-27") + assert(schema.revisions[2].description == "Revision 2") + assert(schema.revisions[2].value == "2016-05-28") + + -- Check that the feature statements are in the exports interface + -- but not the schema itself. + assert(not schema.features) + assert(env.features["bowl"]) + assert(env.features["bowl"].description == 'A fruit bowl') + + -- Check that groupings get inlined into their uses. + assert(schema.body['fruit-bowl']) + assert(schema.body['fruit-bowl'].description == 'Represents a fruit bowl') + local contents = schema.body['fruit-bowl'].body['contents'] + assert(contents) + assert(contents.kind == 'list') + -- TODO: Copy description over? Probably not given that one node + -- can use multiple groupings. + -- assert(contents.description == 'Represents a piece of fruit') + assert(contents.body['name'].kind == 'leaf') + assert(contents.body['name'].type.id == 'string') + assert(contents.body["name"].mandatory == true) + assert(contents.body["name"].description == "Name of fruit.") + assert(contents.body["score"].type.id == "uint8") + assert(contents.body["score"].mandatory == true) + assert(contents.body["score"].type.range.value[1] == 0) + assert(contents.body["score"].type.range.value[2] == 10) + + -- Check the container has a leaf called "description" + local desc = schema.body["fruit-bowl"].body['description'] + assert(desc.type.id == "string") + assert(desc.description == "About the bowl") + + parse_schema(require('lib.yang.ietf_yang_types_yang')) + parse_schema(require('lib.yang.ietf_inet_types_yang')) + + load_schema_by_name('ietf-yang-types') + load_schema_by_name('ietf-softwire') end diff --git a/src/lib/yang/schema2.lua b/src/lib/yang/schema2.lua deleted file mode 100644 index fc5973e5a4..0000000000 --- a/src/lib/yang/schema2.lua +++ /dev/null @@ -1,927 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. --- This module implements the schema tree and validation for YANG. It represents --- the YANG statements with lua tables and provides a fast but flexible way to --- represent and validate statements. --- --- Since YANG statements are encapsulated in modules at the highest level one --- should take their pre-parsed YANG document containing the module and load it --- into the Module table. -module(..., package.seeall) -local parser = require("lib.yang.parser") - -local ffi = require("ffi") -ffi.cdef("long long atoll(const char *nptr);") -local function tointeger(str) - local i = ffi.C.atoll(str) - if tostring(i) == str.."LL" then - if i == tonumber(i) then return tonumber(i) else return i end - end -end - -local function error_with_path(path, msg, ...) - error(string.format("%s: "..msg, path, ...)) -end -local function assert_with_path(expr, path, msg, ...) - if not expr then error_with_path(path, msg, ...) end - return expr -end - --- (kind -> (function(Node) -> value)) -local initializers = {} -local function declare_initializer(init, ...) - for _, keyword in ipairs({...}) do initializers[keyword] = init end -end - -local Node = {} -local function parse_node(src, parent_path, order) - local ret = {} - ret.kind = assert(src.keyword, 'missing keyword') - if parent_path then - ret.path = parent_path..'.'..ret.kind - else - ret.path = ret.kind - end - ret.order = order - ret.argument_string = src.argument - ret.children = parse_children(src, ret.path) - ret = setmetatable(ret, {__index=Node}) - local initialize = initializers[ret.kind] - if initialize then initialize(ret) end - return ret -end - -function parse_children(src, parent_path) - local ret = {} - for i, statement in ipairs(src.statements or {}) do - local child = parse_node(statement, parent_path, i) - if not ret[child.kind] then ret[child.kind] = {} end - table.insert(ret[child.kind], child) - end - return ret -end - -local function require_argument(node) - return assert_with_path(node.argument_string, node.path, - 'missing argument') -end - -local function parse_range(node, range) - local function parse_part(part) - local l, r = part:match("^%s*([^%.]*)%s*%.%.%s*([^%s]*)%s*$") - assert_with_path(l, node.path, 'bad range component: %s', part) - if l ~= 'min' then - l = assert_with_path(tointeger(l), node.path, "bad integer: %s", l) - end - if r ~= 'max' then - r = assert_with_path(tointeger(r), node.path, "bad integer: %s", r) - end - return { l, r } - end - local parts = range:split("|") - local res = {'or'} - for part in range:split("|") do table.insert(res, parse_part(part)) end - if #res == 1 then error_with_path(node.path, "empty range", range) - elseif #res == 2 then return res[2] - else return res end -end - -local function collect_children(node, kinds) - if type(kinds) == 'string' then return collect_children(node, {kinds}) end - local ret = {} - for _, kind in ipairs(kinds) do - if node.children[kind] then - for _, child in pairs(node.children[kind]) do - table.insert(ret, child) - end - end - end - return ret -end - -local function collect_children_by_prop(node, kinds, prop) - local ret = {} - for _, child in ipairs(collect_children(node, kinds)) do - assert_with_path(child[prop], node.path, - 'child of kind %s missing prop %s', child.kind, prop) - assert_with_path(not ret[child[prop]], node.path, - 'duplicate %s: %s', prop, child[prop]) - ret[child[prop]] = child - end - return ret -end - -local function collect_children_by_id(node, kinds) - return collect_children_by_prop(node, kinds, 'id') -end - -local function collect_body_children(node) - return collect_children_by_id( - node, - {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml'}) -end - -local function at_least_one(tab) - for k, v in pairs(tab) do return true end - return false -end - -local function collect_body_children_at_least_1(node) - local ret = collect_body_children(node) - if not at_least_one(ret) then - error_with_path(node.path, "missing data statements") - end - return ret -end - -local function collect_data_or_case_children_at_least_1(node) - local ret = collect_children_by_id( - node, - {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', - 'anyxml', 'case'}) - if not at_least_one(ret) then - error_with_path(node.path, "missing data statements") - end - return ret -end - -local function collect_child_properties(node, kind, field) - local ret = {} - for _, child in ipairs(collect_children(node, kind)) do - table.insert(ret, child[field]) - end - return ret -end - -local function maybe_child(node, kind) - local children = collect_children(node, kind) - if #children > 1 then - error_with_path(node.path, 'expected at most one child of type %s', kind) - end - return children[1] -end - -local function maybe_child_property(node, kind, prop) - local child = maybe_child(node, kind) - if child then return child[prop] end -end - -local function require_child(node, kind) - local child = maybe_child(node, kind) - if child then return child end - error_with_path(node.path, 'missing child of type %s', kind) -end - -local function require_child_property(node, kind, prop) - return require_child(node, kind)[prop] -end - --- Simple statement kinds with string, natural, or boolean values all --- just initialize by parsing their argument and storing it as the --- "value" property in the schema node. -local function init_string(node) - node.value = require_argument(node) -end -local function init_natural(node) - local arg = require_argument(node) - local as_num = tonumber(arg) - assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, - node.path, 'not a natural number: %s', arg) - node.value = as_num -end -local function init_boolean(node) - local arg = require_argument(node) - if arg == 'true' then node.value = true - elseif arg == 'false' then node.value = false - else error_with_path(node.path, 'not a valid boolean: %s', arg) end -end - --- For all other statement kinds, we have custom initializers that --- parse out relevant sub-components and store them as named --- properties on the schema node. -local function init_anyxml(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.must = collect_child_properties(node, 'must', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_argument(node) - node.id = require_argument(node) - node.yin_element = maybe_child_property(node, 'yin-element', 'value') -end -local function init_augment(node) - node.node_id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.body = collect_data_or_case_children_at_least_1(node) -end -local function init_belongs_to(node) - node.id = require_argument(node) - node.prefix = require_child(node, 'prefix').value -end -local function init_case(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.body = collect_body_children(node) -end -local function init_choice(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.default = maybe_child_property(node, 'default', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_children_by_id( - node, - {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) -end -local function init_container(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.must = collect_child_properties(node, 'must', 'value') - node.presence = maybe_child_property(node, 'presence', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_body_children(node) -end -local function init_extension(node) - node.id = require_argument(node) - node.argument = maybe_child_property(node, 'argument', 'id') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_feature(node) - node.id = require_argument(node) - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_grouping(node) - node.id = require_argument(node) - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_body_children(node) -end -local function init_identity(node) - node.id = require_argument(node) - node.base = maybe_child_property(node, 'base', 'id') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_import(node) - node.id = require_argument(node) - node.prefix = require_child_property(node, 'prefix', 'value') - node.revision_date = maybe_child_property(node, 'revision-date', 'value') -end -local function init_include(node) - node.id = require_argument(node) - node.revision_date = maybe_child_property(node, 'revision-date', 'value') -end -local function init_input(node) - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_body_children_at_least_1(node) -end -local function init_leaf(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.type = require_child(node, 'type') - node.units = maybe_child_property(node, 'units', 'value') - node.must = collect_child_properties(node, 'must', 'value') - node.default = maybe_child_property(node, 'default', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_leaf_list(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.type = require_child(node, 'type') - node.units = maybe_child_property(node, 'units', 'value') - node.must = collect_child_properties(node, 'must', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.min_elements = maybe_child_property(node, 'min-elements', 'value') - node.max_elements = maybe_child_property(node, 'max-elements', 'value') - node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_length(node) - -- TODO: parse length arg str - node.value = require_argument(node) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_list(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.must = collect_child_properties(node, 'must', 'value') - node.key = maybe_child_property(node, 'key', 'value') - node.unique = collect_child_properties(node, 'unique', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.min_elements = maybe_child_property(node, 'min-elements', 'value') - node.max_elements = maybe_child_property(node, 'max-elements', 'value') - node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_body_children_at_least_1(node) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_module(node) - node.id = require_argument(node) - node.yang_version = maybe_child_property(node, 'yang-version', 'value') - node.namespace = require_child_property(node, 'namespace', 'value') - node.prefix = require_child_property(node, 'prefix', 'value') - node.imports = collect_children_by_prop(node, 'import', 'prefix') - node.includes = collect_children_by_id(node, 'include') - node.organization = maybe_child_property(node, 'organization', 'value') - node.contact = maybe_child_property(node, 'contact', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.revisions = collect_children(node, 'revision') - node.augments = collect_children(node, 'augment') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.features = collect_children_by_id(node, 'feature') - node.extensions = collect_children_by_id(node, 'extension') - node.identities = collect_children_by_id(node, 'identity') - node.rpcs = collect_children_by_id(node, 'rpc') - node.notifications = collect_children_by_id(node, 'notification') - node.deviations = collect_children_by_id(node, 'deviation') - node.body = collect_body_children(node) -end -local function init_namespace(node) - -- TODO: parse uri? - node.value = require_argument(node) -end -local function init_notification(node) - node.id = require_argument(node) - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_body_children(node) -end -local function init_output(node) - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.body = collect_body_children_at_least_1(node) -end -local function init_path(node) - -- TODO: parse path string - node.value = require_argument(node) -end -local function init_pattern(node) - node.value = require_argument(node) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_range(node) - node.value = parse_range(node, require_argument(node)) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_refine(node) - node.node_id = require_argument(node) - -- All subnode kinds. - node.must = collect_child_properties(node, 'must', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - -- Containers. - node.presence = maybe_child_property(node, 'presence', 'value') - -- Leaves, choice, and (for mandatory) anyxml. - node.default = maybe_child_property(node, 'default', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') - -- Leaf lists and lists. - node.min_elements = maybe_child_property(node, 'min-elements', 'value') - node.max_elements = maybe_child_property(node, 'max-elements', 'value') -end -local function init_revision(node) - -- TODO: parse date - node.value = require_argument(node) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_rpc(node) - node.id = require_argument(node) - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.input = maybe_child(node, 'input') - node.output = maybe_child(node, 'output') -end -local function init_type(node) - node.id = require_argument(node) - node.range = maybe_child(node, 'range') - node.fraction_digits = maybe_child_property(node, 'fraction-digits', 'value') - node.length = maybe_child_property(node, 'length', 'value') - node.patterns = collect_children(node, 'pattern') - node.enums = collect_children(node, 'enum') - -- !!! path - node.leafref = maybe_child_property(node, 'path', 'value') - node.require_instances = collect_children(node, 'require-instance') - node.identityref = maybe_child_property(node, 'base', 'value') - node.union = collect_children(node, 'type') - node.bits = collect_children(node, 'bit') -end -local function init_submodule(node) - node.id = require_argument(node) - node.yang_version = maybe_child_property(node, 'yang-version', 'value') - node.belongs_to = require_child(node, 'belongs-to') - node.imports = collect_children_by_id(node, 'import') - node.includes = collect_children_by_id(node, 'include') - node.organization = maybe_child_property(node, 'organization', 'value') - node.contact = maybe_child_property(node, 'contact', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.revisions = collect_children(node, 'revision') - node.augments = collect_children(node, 'augment') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.features = collect_children_by_id(node, 'feature') - node.extensions = collect_children_by_id(node, 'extension') - node.identities = collect_children_by_id(node, 'identity') - node.rpcs = collect_children_by_id(node, 'rpc') - node.notifications = collect_children_by_id(node, 'notification') - node.deviations = collect_children_by_id(node, 'deviation') - node.body = collect_body_children(node) -end -local function init_typedef(node) - node.id = require_argument(node) - node.type = require_child(node, 'type') - node.units = maybe_child_property(node, 'units', 'value') - node.default = maybe_child_property(node, 'default', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') -end -local function init_uses(node) - node.id = require_argument(node) - node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.refines = collect_children(node, 'refine') - node.augments = collect_children(node, 'augment') -end -local function init_value(node) - local arg = require_argument(node) - local as_num = tonumber(arg) - assert_with_path(as_num and math.floor(as_num) == as_num, - node.path, 'not an integer: %s', arg) - node.value = as_num -end - -declare_initializer( - init_string, 'prefix', 'organization', 'contact', 'description', - 'reference', 'units', 'revision-date', 'base','if-feature', - 'default', 'enum', 'bit', 'status', 'presence', 'ordered-by', 'must', - 'error-message', 'error-app-tag', 'max-value', 'key', 'unique', 'when', - 'deviation', 'deviate') -declare_initializer( - init_natural, 'yang-version', 'fraction-digits', 'position', - 'min-elements', 'max-elements') -declare_initializer( - init_boolean, 'config', 'mandatory', 'require-instance', 'yin-element') -declare_initializer(init_anyxml, 'anyxml') -declare_initializer(init_argument, 'argument') -declare_initializer(init_augment, 'augment') -declare_initializer(init_belongs_to, 'belongs-to') -declare_initializer(init_case, 'case') -declare_initializer(init_choice, 'choice') -declare_initializer(init_container, 'container') -declare_initializer(init_extension, 'extension') -declare_initializer(init_feature, 'feature') -declare_initializer(init_grouping, 'grouping') -declare_initializer(init_identity, 'identity') -declare_initializer(init_import, 'import') -declare_initializer(init_include, 'include') -declare_initializer(init_input, 'input') -declare_initializer(init_leaf, 'leaf') -declare_initializer(init_leaf_list, 'leaf-list') -declare_initializer(init_length, 'length') -declare_initializer(init_list, 'list') -declare_initializer(init_module, 'module') -declare_initializer(init_namespace, 'namespace') -declare_initializer(init_notification, 'notification') -declare_initializer(init_output, 'output') -declare_initializer(init_path, 'path') -declare_initializer(init_pattern, 'pattern') -declare_initializer(init_range, 'range') -declare_initializer(init_refine, 'refine') -declare_initializer(init_revision, 'revision') -declare_initializer(init_rpc, 'rpc') -declare_initializer(init_submodule, 'submodule') -declare_initializer(init_type, 'type') -declare_initializer(init_typedef, 'typedef') -declare_initializer(init_uses, 'uses') -declare_initializer(init_value, 'value') - -local function schema_from_ast(ast) - local ret - local submodules = {} - for _,node in ipairs(ast) do - if node.keyword == 'module' then - assert(not ret, 'expected only one module form') - ret = parse_node(node) - elseif node.keyword == 'submodule' then - assert(not submodules[node.id], 'duplicate submodule name: '..node.id) - submodules[node.id] = parse_node(node) - else - error('expected only module and submodule statements, got: '..node.keyword) - end - end - assert(ret, 'missing module form') - ret.submodules = submodules - return ret -end - --- Strip properties pertaining to original source representation. -local function strip(exp) - if type(exp) ~= 'table' then return exp end - local ret = {} - for k, v in pairs(exp) do - if k ~= 'children' and k ~= 'argument_string' and k ~= 'order' and k ~= 'path' then - ret[k] = strip(v) - end - end - return ret -end - -local function set(...) - local ret = {} - for k, v in pairs({...}) do ret[v] = true end - return ret -end - -local primitive_types = set( - 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', - 'binary', 'bits', 'boolean', 'decimal64', 'empty', 'enumeration', - 'identityref', 'instance-identifier', 'leafref', 'string', 'union') - --- Inline "grouping" into "uses". --- Inline "submodule" into "include". --- Inline "imports" into "module". --- Inline "typedef" into "type". (TODO) --- Resolve if-feature. --- Warn on any "when", resolving them as being true. --- Resolve all augment and refine nodes. (TODO) -function resolve(schema, features) - local function shallow_copy(node) - local out = {} - for k,v in pairs(node) do out[k] = v end - return out - end - local function pop_prop(node, prop) - local val = node[prop] - node[prop] = nil - return val - end - local function lookup(env, prop, name) - if not env then error(prop..' not found: '..name) end - if not env[prop] or not env[prop][name] then - return lookup(env.env, prop, name) - end - return env[prop][name] - end - local function lookup_lazy(env, prop, name) - local val = lookup(env, prop, name) - if type(val) == 'table' then return val end - -- Force lazy expansion and memoize result. - return val() - end - local visit - local function visit_top_level(node, env, prop) - assert(not env[prop]) - env[prop] = {} - local p = lookup(env, 'prefix', '_') - for k,v in pairs(pop_prop(node, prop) or {}) do - env[prop][k] = visit(v, env) - env[prop][p..':'..k] = env[prop][k] - end - end - local function visit_lazy(tab, env) - local ret = {} - local prefix = lookup(env, 'prefix', '_') - local function error_recursion() - end - for k,v in pairs(tab) do - -- FIXME: Only add prefix:k if at top level. - local state - local function lazy() - if state == 'visiting' then - error('mutually recursive definitions: '..k) - elseif state then - return state - else - state = 'visiting' - end - state = visit(v, env) - return state - end - ret[k] = lazy - ret[prefix..':'..k] = ret[k] - end - return ret - end - function visit_type(node, env) - node = shallow_copy(node) - local success, typedef = pcall(lookup, env, 'typedefs', node.id) - if success then - -- Could be that typedef is still lazy. We didn't use - -- lookup_lazy because we don't want the pcall to hide errors - -- from the lazy expansion. - if type(typedef) == 'function' then typedef = typedef() end - node.base_type = typedef - else - -- If the type name wasn't bound, it must be primitive. - assert(primitive_types[node.id], 'unknown type: '..node.id) - node.base_type = node.id - end - return node - end - function visit(node, env) - node = shallow_copy(node) - env = {env=env} - if node.typedefs then - -- Populate node.typedefs as a table of thunks that will - -- lazily expand and memoize their result when called. This - -- is not only a performance optimization but also allows the - -- typedefs to be mutually visible. - env.typedefs = visit_lazy(pop_prop(node, 'typedefs'), env) - end - if node.groupings then - -- Likewise expand groupings at their point of definition. - env.groupings = visit_lazy(pop_prop(node, 'groupings'), env) - end - local when = pop_prop(node, 'when') - if when then - print('warning: assuming "when" condition to be true: '..when.value) - end - if node.kind == 'module' or node.kind == 'submodule' then - visit_top_level(node, env, 'extensions') - visit_top_level(node, env, 'features') - visit_top_level(node, env, 'identities') - end - for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do - if not pcall(lookup, env, 'features', feature) then - return nil, env - end - end - if node.type then node.type = visit_type(node.type, env) end - for k,v in pairs(node.body or {}) do - if v.kind == 'uses' then - -- Inline "grouping" into "uses". - local grouping = lookup_lazy(env, 'groupings', v.id) - node.body[k] = nil - for k,v in pairs(grouping.body) do - assert(not node.body[k], 'duplicate identifier: '..k) - node.body[k] = v - end - -- TODO: Handle refine and augment statements. - else - node.body[k] = visit(v, env) - end - end - return node, env - end - local function include(dst, src) - for k,v in pairs(src) do - assert(dst[k] == nil or dst[k] == v, 'incompatible definitions: '..k) - if not k:match(':') then dst[k] = v end - end - end - local linked = {} - local function link(node, env) - if linked[node.id] then - assert(linked[node.id] ~= 'pending', 'recursive import of '..node.id) - local node, env = unpack(linked[node.id]) - return node, env - end - linked[node.id] = 'pending' - node = shallow_copy(node) - local module_env = {env=env, prefixes={}, extensions={}, features={}, - identities={}, typedefs={}, groupings={}} - local module_body = shallow_copy(node.body) - for k,v in pairs(pop_prop(node, 'includes')) do - local submodule = lookup(env, 'submodules', k) - assert(submodule.belongs_to.id == node.id) - submodule, submodule_env = link(submodule, env) - include(module_env.extensions, submodule_env.extensions) - include(module_env.features, submodule_env.features) - include(module_env.identities, submodule_env.identities) - include(module_env.typedefs, submodule_env.typedefs) - include(module_env.groupings, submodule_env.groupings) - include(module_body, submodule.body) - end - if node.prefix then - assert(node.kind == 'module', node.kind) - module_env.prefixes[node.prefix] = node.namespace - module_env.prefix = {_=node.prefix} - end - for k,v in pairs(pop_prop(node, 'imports')) do - assert(not module_env.prefixes[v.prefix], 'duplicate prefix') - -- CHECKME: Discarding body from import, just importing env. - -- Is this OK? - local schema, env = load_schema_by_name(v.id, v.revision_date) - local prefix = v.prefix - module_env.prefixes[prefix] = schema.namespace - for _,prop in ipairs({'extensions', 'features', 'identities', - 'typedefs', 'groupings'}) do - for k,v in pairs(env[prop]) do - if not k:match(':') then - module_env[prop][prefix..':'..k] = v - end - end - end - end - node, env = visit(node, module_env) - -- The typedefs, groupings, identities, and so on of this module - -- are externally visible for other modules that may import this - -- one; save them and their environment. - linked[node.id] = {node, env} - return node, env - end - schema = shallow_copy(schema) - return link(schema, {features=(features or {}), - submodules=pop_prop(schema, 'submodules')}) -end - -function parse_schema(src, filename) - return schema_from_ast(parser.parse_string(src, filename)) -end -function parse_schema_file(filename) - return schema_from_ast(parser.parse_file(filename)) -end - -function load_schema(src, filename) - return resolve(strip(parse_schema(src, filename))) -end -function load_schema_file(filename) - return resolve(strip(parse_schema_file(filename))) -end -function load_schema_by_name(name, revision) - -- FIXME: @ is not valid in a Lua module name. - -- if revision then name = name .. '@' .. revision end - name = name:gsub('-', '_') - return load_schema(require('lib.yang.'..name..'_yang'), name) -end - -function selftest() - local test_schema = [[module fruit { - namespace "urn:testing:fruit"; - prefix "fruit"; - - import ietf-inet-types {prefix inet; } - import ietf-yang-types {prefix yang; } - - organization "Fruit Inc."; - - contact "John Smith fake@person.tld"; - - description "Module to test YANG schema lib"; - - revision 2016-05-27 { - description "Revision 1"; - reference "tbc"; - } - - revision 2016-05-28 { - description "Revision 2"; - reference "tbc"; - } - - feature bowl { - description "A fruit bowl"; - reference "fruit-bowl"; - } - - grouping fruit { - description "Represents a piece of fruit"; - - leaf name { - type string; - mandatory true; - description "Name of fruit."; - } - - leaf score { - type uint8 { - range 0..10; - } - mandatory true; - description "How nice is it out of 10"; - } - - leaf tree-grown { - type boolean; - description "Is it grown on a tree?"; - } - } - - container fruit-bowl { - description "Represents a fruit bowl"; - - leaf description { - type string; - description "About the bowl"; - } - - list contents { - uses fruit; - } - } - }]] - - local schema, env = load_schema(test_schema) - assert(schema.id == "fruit") - assert(schema.namespace == "urn:testing:fruit") - assert(schema.prefix == "fruit") - assert(schema.contact == "John Smith fake@person.tld") - assert(schema.organization == "Fruit Inc.") - assert(schema.description == "Module to test YANG schema lib") - - -- Check all revisions are accounted for. - assert(schema.revisions[1].description == "Revision 1") - assert(schema.revisions[1].value == "2016-05-27") - assert(schema.revisions[2].description == "Revision 2") - assert(schema.revisions[2].value == "2016-05-28") - - -- Check that the feature statements are in the exports interface - -- but not the schema itself. - assert(not schema.features) - assert(env.features["bowl"]) - assert(env.features["bowl"].description == 'A fruit bowl') - - -- Check that groupings get inlined into their uses. - assert(schema.body['fruit-bowl']) - assert(schema.body['fruit-bowl'].description == 'Represents a fruit bowl') - local contents = schema.body['fruit-bowl'].body['contents'] - assert(contents) - assert(contents.kind == 'list') - -- TODO: Copy description over? Probably not given that one node - -- can use multiple groupings. - -- assert(contents.description == 'Represents a piece of fruit') - assert(contents.body['name'].kind == 'leaf') - assert(contents.body['name'].type.id == 'string') - assert(contents.body["name"].mandatory == true) - assert(contents.body["name"].description == "Name of fruit.") - assert(contents.body["score"].type.id == "uint8") - assert(contents.body["score"].mandatory == true) - assert(contents.body["score"].type.range.value[1] == 0) - assert(contents.body["score"].type.range.value[2] == 10) - - -- Check the container has a leaf called "description" - local desc = schema.body["fruit-bowl"].body['description'] - assert(desc.type.id == "string") - assert(desc.description == "About the bowl") - - parse_schema(require('lib.yang.ietf_yang_types_yang')) - parse_schema(require('lib.yang.ietf_inet_types_yang')) - - load_schema_by_name('ietf-yang-types') - load_schema_by_name('ietf-softwire') -end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 489f39f510..8837ab9f1b 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -1,415 +1,15 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. --- This is the entry-point to the lua YANG library. It will handle parsing a --- file with YANG statements and producing a schema tree and data tree. --- --- To use this you should use the helper functions: --- - load_schema --- - load_schema_file --- The former takes in a yang string and builds a Base table, similarly the --- load_schema_file takes a schema file and builds a Base table from that. --- --- Once you have the base file you can inspect the schema, data and set values. --- Example: --- local base = load_schema[[ --- module example { --- namespace "exampleinc:example"; --- prefix "example"; --- organization "Example Inc."; --- description "An example document" --- --- container conf { --- leaf description { --- type string; --- } --- leaf testvalue { --- type uint8; --- } --- } --- }]] --- base["example"].organization == "YANG IPv4 over IPv6" --- base["example"].containers["conf"].leaves["description"].type == "string" --- --- -- Setting data --- base.data["example"]["conf"].description = "hello!" --- base.data["example"]["conf"].testvalue = "error" -- This errors. --- base.data["example"]["conf"].testvalue = 50000 -- Beware: they overflow. module(..., package.seeall) local schema = require("lib.yang.schema") -local helpers = require("lib.yang.helpers") -local parser = require("lib.yang.parser") -local Container = helpers.Container -local asserterror = helpers.asserterror -local setvalue = helpers.setvalue +local data = require("lib.yang.data") -Base = {} -function Base.new(filename) - local ret = {schema={}, filename=filename} - local self = setmetatable(ret, {__index=Base, path_cache={}}) - self.data = Container.new(self, "") - return self -end - -function Base:error(path, node, msg, ...) - error(("Error: %s.%s: %s"):format(path, node, (msg):format(...))) -end - -function Base:load(src) - src = helpers.extract_nodes(src) - if not src.module then - error(("%s: Expected 'module'"):format(self.filename)) - end - local mod = schema.Module.new( - self, - src.module[1].argument, - src.module[1].statements - ) - self.schema[src.module[1].argument] = mod - - self.data:add_container(mod.name) - return self.schema -end - -function Base:get_schema(path) - -- Handle case when path is empty. (Provide with self.data) - if path == "" then - return self.data - end - - -- Look in the cache and hopefully return the schema node for the path. - local cache = getmetatable(self).path_cache - return cache[path] -end - -function Base:add_cache(path, node) - getmetatable(self).path_cache[path] = node -end - -function Base:get_module() - local schema_mod - for _, mod in pairs(self.schema) do - schema_mod = mod - end - - if not schema_mod then - error("Module cannot be resolved") - end - - return schema_mod -end - -function Base:produce_data_tree(schema_node, data_node) - if not (schema_node and data_node) then - schema_node = self:get_module() - - if not schema_node then error("Module cannot be resolved") end - data_node = self.data[schema_node.name] - end - local path = getmetatable(schema_node).path - - if schema_node.containers then - for name, container in pairs(schema_node.containers) do - local new_path = path.."."..name - local new_node = Container.new(self, new_path, data_node) - - local current_schema = assert(self:get_schema(new_path), "No schema at path:"..new_path) - - data_node:add_to_root(name, new_node) - self:produce_data_tree(current_schema, new_node) - - -- If the container has a "uses" statement we must copy across the - -- leaves from the container it references to this container. - if container.uses then - self:handle_use(container, data_node, path, name) - end - end - end - - if schema_node.leaves then - for name, leaf in pairs(schema_node.leaves) do - -- Certain types need extra options passing in, depending on the type those - -- need to be passed in when creating the type. - local options = nil - if leaf.type == "enumeration" then - options = leaf.enums - elseif leaf.type == "union" then - options = leaf.types - end - data_node:add_to_root(name, helpers.create_box(leaf.type, options, leaf.default)) - end - end - - if schema_node.lists then - for name, list in pairs(schema_node.lists) do - local list_path = path.."."..name - local container = Container.new(self, list_path, data_node) - local current_schema = assert(self:get_schema(list_path), "No schema at path:"..list_path) - data_node:add_to_root(name, container) - - if list.uses then - local template = Container.new(self, list_path, data_node) - self:handle_use(list, template, list_path, name) - container:set_template(template) - else - -- Make a data container for the template - local template_container = Container.new(self, list_path) - local data_template = self:produce_data_tree( - current_schema, - template_container - ) - - container:set_template(template_container) - end - end - end - - return data_node -end - -function Base:schema_for_uses(schema) - if schema.uses == nil then - error("Can only find schema for a node which uses the `use` statement.") - end - - return self:get_module().groupings[schema.uses] -end - -function Base:handle_use(schema_node, data_node, path, name) - local grouping = self:schema_for_uses(schema_node) - if not grouping then - self:error(path, name, "Cannot find grouping '%s'.", schema_node.uses) - end - - -- Copy. - for name, leaf in pairs(grouping.leaves) do - -- We also need to register the schema node at the new path - local grouping_path = path.."."..name - self:add_cache(grouping_path, leaf) - data_node:add_to_root(name, helpers.create_box(leaf.type, leaf.default)) - end -end - -function Base:add(key, node) - self.data[key] = node -end - -function Base:load_data(data, filename) - local parsed_data = parser.parse_string(data, filename) - local data = self:produce_data_tree() - - -- Function which can take a node, set any leaf values and recursively call - -- over collections, etc. - function recursively_add(node, path, parent, data_node) - for _, n in pairs(node) do - -- Create a path for the keyword argument pair. - local new_path = "" - if path == nil then - new_path = n.keyword - else - new_path = path.."."..n.keyword - end - - local schema = self:get_schema(new_path) - if n.statements then - if schema.get_type() == "list" then - -- Lists are a special case, first we need to conver them from their - -- current format to a more useful {leafname: value, leafname: value} - -- type table. Then we want to add the item to the list parent. - local converted = {} - for _, leaf in pairs(n.statements) do - converted[leaf.keyword] = leaf.argument - end +load_schema = schema.load_schema +load_schema_file = schema.load_schema_file +load_schema_by_name = schema.load_schema_by_name - -- Extract key and add it to the converted. - converted[schema.key] = n.argument - - local data_node = self:find_data_node(new_path, nil, nil, true) - data_node:add_item(converted) - - recursively_add(n.statements, new_path, data_node[n.argument]) - else - if parent ~= nil then - local dn = parent[n.keyword] - recursively_add(n.statements, new_path, parent, dn) - else - recursively_add(n.statements, new_path) - end - end - else - if data_node == nil then - data_node = self:find_data_node(path) - end - data_node[n.keyword] = n.argument - end - end - end - - -- Recursively add the data - recursively_add(parsed_data) - return self.data -end - -function Base:find_data_node(schema, data_node, current_schema, raw) - -- If no data node has been provided assume we're starting from Base.data - if data_node == nil then - data_node = self.data - end - - -- If there is no current_schema we must be in our first iteration (or - -- someone has called this function incorrectly). - if current_schema == nil then - current_schema = schema - end - - -- First find the first node in the schema - local head = current_schema:match("[%w-_]*") - local tail = current_schema:sub(head:len() + 2) - - -- If it's a list we want to access the list's version as that's where all - -- the values are, the non-templated version is just the raw data. - local node - if data_node:get_template() ~= nil then - node = data_node:get_template()[head] - else - node = data_node[head] - end - - -- If the node doesn't exist, we should display a useful error. - if node == nil then - local current = nil - if current_schema then - current = current_schema - else - current = schema - end - self:error(current, head, "Can't find data node") - end - - -- Otherwise we need to check if we're at the end of the schema, if we are - -- we should return what we've found, otherwise continue to recursively call. - if tail == "" then - if raw ~= true and node:get_template() ~= nil then - return node:get_template() - else - return node - end - else - return self:find_data_node(schema, node, tail, raw) - end -end - -function Base:load_data_file(filename) - local file_in = assert(io.open(filename)) - local contents = file_in:read("*a") - file_in:close() - return self:load_data(contents, filename) -end - -function load_schema(schema, filename) - local parsed_yang = parser.parse_string(schema, "selftest") - local base = Base.new() - base:load(parsed_yang) - return base -end - -function load_schema_file(filename) - local file_in = assert(io.open(filename)) - local contents = file_in:read("*a") - file_in:close() - return load_schema(contents, filename) -end +load_data_for_schema = data.load_data_for_schema +load_data_for_schema_by_name = data.load_data_for_schema_by_name function selftest() - local test_schema = [[module fruit { - namespace "urn:testing:fruit"; - prefix "fruit"; - - import ietf-inet-types {prefix inet; } - import ietf-yang-types {prefix yang; } - - organization "Fruit Inc."; - - contact "John Smith fake@person.tld"; - - description "Module to test YANG schema lib"; - - revision 2016-05-27 { - description "Revision 1"; - reference "tbc"; - } - - revision 2016-05-28 { - description "Revision 2"; - reference "tbc"; - } - - feature bowl { - description "A fruit bowl"; - reference "fruit-bowl"; - } - - grouping fruit { - description "Represets a piece of fruit"; - - leaf name { - type string; - mandatory true; - description "Name of fruit."; - } - - leaf score { - type uint8 { - range 0..10; - } - mandatory true; - description "How nice is it out of 10"; - } - - leaf tree-grown { - type boolean; - description "Is it grown on a tree?"; - } - } - - container fruit-bowl { - description "Represents a fruit bowl"; - - leaf description { - type string; - description "About the bowl"; - } - - list contents { - uses fruit; - } - } - }]] - local base = load_schema(test_schema) - local data = base:produce_data_tree() - local bowl = data.fruit["fruit-bowl"] - - bowl.description = "hello!" - assert(bowl.description == "hello!") - - -- Add items to fruit-bowl's contents list. - bowl.contents:add_item({name="Banana", score=10}) - assert(bowl.contents[1].name == "Banana") - assert(bowl.contents[1].score == 10) - - -- Check that validation still works in lists with groupings. - asserterror(setvalue, bowl.contents[1].score, "fail") - - -- Check that an entry can't be added with missing required fields - asserterror(bowl.contents.add_item, bowl.contents, {score=10}) - - -- Check that an entry with incorrect data can't be added. - asserterror( - bowl.contents.add_item, - bowl.contents, - {name="Pear", score=5, invalid=true} - ) - - -- Finally check tht validation occurs when you're adding entries with - -- invalid data in them, in this case sore needs to be an integer. - asserterror(bowl.contents, bowl.contents, {name="Pear", score="Good"}) end From 70f36372d7b44414feb6d5732bf479df37c1668f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 14:44:59 +0100 Subject: [PATCH 079/631] Internal yang refactor * src/lib/yang/util.lua: Rename from handlers.lua. Drop dead code and incorporate tointeger. * src/lib/yang/schema.lua: * src/lib/yang/data.lua: Use util.lua for tointeger. --- src/lib/yang/data.lua | 36 +-- src/lib/yang/helpers.lua | 560 --------------------------------------- src/lib/yang/schema.lua | 26 +- src/lib/yang/util.lua | 58 ++++ 4 files changed, 71 insertions(+), 609 deletions(-) delete mode 100644 src/lib/yang/helpers.lua create mode 100644 src/lib/yang/util.lua diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 7eb8144709..42eff71a60 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -2,9 +2,16 @@ -- COPYING. module(..., package.seeall) -local ffi = require("ffi") local parse_string = require("lib.yang.parser").parse_string local schema = require("lib.yang.schema") +local util = require("lib.yang.util") + +-- FIXME: +-- Parse inet:mac-address using ethernet:pton +-- Parse inet:ipv4-address using ipv4:pton +-- Parse inet:ipv6-address using ipv6:pton +-- Parse inet:ipv4-prefix? +-- Parse inet:ipv6-prefix? function data_grammar_from_schema(schema) local handlers = {} @@ -42,34 +49,9 @@ function data_grammar_from_schema(schema) return {type="struct", members=visit_body(schema)} end -ffi.cdef([[ -unsigned long long strtoull (const char *nptr, const char **endptr, int base); -]]) - local function integer_type(min, max) return function(str, k) - local str = assert(str, 'missing value for '..k) - local start = 1 - local is_negative - local base = 10 - if str:match('^-') then start, is_negative = 2, true - elseif str:match('^+') then start = 2 end - if str:match('^0x', start) then base, start = 16, start + 2 - elseif str:match('^0', start) then base = 8 end - str = str:lower() - local function check(test) - return assert(test, 'invalid numeric value for '..k..': '..str) - end - check(start <= str:len()) - -- FIXME: check that res did not overflow the 64-bit number - local res = ffi.C.strtoull(str:sub(start), nil, base) - if is_negative then - res = ffi.new('int64_t[1]', -1*res)[0] - check(res <= 0) - end - check(min <= res and res <= max) - if tonumber(res) == res then return tonumber(res) end - return res + return util.tointeger(str, k, min, max) end end diff --git a/src/lib/yang/helpers.lua b/src/lib/yang/helpers.lua deleted file mode 100644 index adccab4ef1..0000000000 --- a/src/lib/yang/helpers.lua +++ /dev/null @@ -1,560 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. -module(..., package.seeall) - -local h = require("syscall.helpers") -local ffi = require("ffi") -local ipv4 = require("lib.protocol.ipv4") -local ipv6 = require("lib.protocol.ipv6") -local corelib = require("core.lib") -local ethernet = require("lib.protocol.ethernet") - --- Create small getter and setter wrapper for ffi structs. -local FFIType = {} - -function FFIType:set(value) - self.value = convert(value) -end - -function FFIType:get() - return self.value -end - - --- Converts a type to a native lua type, this will only work for things we've coded --- to convert (i.e. it's not general purpose). -function convert(value) - -- First try boolean. - if value == "true" then - return true - elseif value == "false" then - return false - else - return tonumber(value) -- returns nil if no number is found - end -end - --- This wraps the FFIType to provide it with the box name, - --- Use ffi types because they will validate that numeric values are being --- provided. The downside is that integer overflow could occur on these. This --- route has been selected as validation will be faster than attempting to --- validate in Lua. -local box_types = { - int8 = ffi.typeof("struct { int8_t value; }"), - int16 = ffi.typeof("struct { int16_t value; }"), - int32 = ffi.typeof("struct { int32_t value; }"), - int64 = ffi.typeof("struct { int64_t value; }"), - uint8 = ffi.typeof("struct { uint8_t value; }"), - uint16 = ffi.typeof("struct { uint16_t value; }"), - uint32 = ffi.typeof("struct { uint32_t value; }"), - uint64 = ffi.typeof("struct { uint64_t value; }"), - decimal64 = ffi.typeof("struct { double value; }"), - boolean = ffi.typeof("struct { bool value; }") -} - --- Iterate through the boxes and set the FFIType metatype -for n, box in pairs(box_types) do - ffi.metatype(box, {__index=FFIType,}) -end - --- Add inet types, found: https://tools.ietf.org/html/rfc6021 --- Should be done via yang module import but using ad-hoc method for now. -box_types["yang:zero-based-counter64"] = function() - return box_types.uint64(0) -end - --- Add the types defined in the snabb-softwire yang module. This should be --- handled by consumption of the 'typedef' statements in the future, however --- for now, hard code them in. -box_types["PacketPolicy"] = function() - local enum_opts = {"allow", "deny"} - return box_types.enumeration(enum_opts) -end - --- WARNING: This doesn't check the range as defined in the config -box_types["PositiveNumber"] = function(...) - return box_types.uint32() -end - --- WARNING: This doesn't check the range as defined in the config -box_types["VlanTag"] = function(...) - return box_types.uint16() -end - -function create_box(leaf_type, arguments) - local box = assert(box_types[leaf_type], "Unsupported type: "..leaf_type) - if box and arguments ~= nil then - box = box(arguments) - elseif box then - box = box() - end - return box -end - -function extract_nodes(schema) - -- This function takes a table which is in the format: - -- {1={keyword="leaf", statements={...}}} - -- and converts this to a more useful easy to access: - -- {leaf={1={...}}} - - local nodes = {} - for _, v in pairs(schema) do - -- Node has statements (children) so we should recursively apply the - -- `extract_nodes` function to them so the entire tree is extracted - if v.statements then v.statements = extract_nodes(v.statements) end - if nodes[v.keyword] then - table.insert(nodes[v.keyword], v) - elseif v.keyword then - nodes[v.keyword] = {v} - end - end - return nodes -end - -local StringBox = {} -function StringBox.new(options, default) - local strbox = setmetatable({}, {__index=StringBox}) - if default ~= nil then - strbox:set(default) - end - return strbox -end - -function StringBox:get() - return self.value -end - -function StringBox:set(value) - self.value = value -end -function StringBox.get_type() return "box-string" end -box_types["string"] = StringBox.new - -local MacAddress = {} -function MacAddress.new(options, default) - local mac = setmetatable({}, {__index=MacAddress}) - if default ~= nil then - mac:set(default) - end - return mac -end -function MacAddress:get() - return self.box -end -function MacAddress:set(address) - self.box = assert(ethernet:pton(address)) -end -function MacAddress.get_type() return "box-inet:mac-address" end -box_types["inet:mac-address"] = MacAddress.new - -local IPv4Box = {} -function IPv4Box.new(options, default) - local ipv4 = setmetatable({}, {__index=IPv4Box}) - if default ~= nil then - ipv4:set(default) - end - return ipv4 -end -function IPv4Box:get() - return self.box -end -function IPv4Box:set(address) - self.box = assert(ipv4:pton(address)) -end -function IPv4Box.get_type() return "box-inet:ipv4-address" end -box_types["inet:ipv4-address"] = IPv4Box.new - -local IPv6Box = {} -function IPv6Box.new(options, default) - local ipv6 = setmetatable({}, {__index=IPv6Box}) - if default ~= nil then - ipv6:set(default) - end - return ipv6 -end -function IPv6Box:get() - return self.box -end -function IPv6Box:set(address) - self.box = assert(ipv6:pton(address)) -end -function IPv6Box.get_type() return "box-inet:ipv6-address" end -box_types["inet:ipv6-address"] = IPv6Box.new - -local IPv4PrefixBox = {} -function IPv4PrefixBox.new(options, default) - local ret = {root={}} - local ipv4_prefix = setmetatable(ret, {__index=IPv4PrefixBox}) - if default ~= nil then - ipv4_prefix:set(default) - end - return ipv4_prefix -end - -function IPv4PrefixBox:get() - return self.value -end -box_types["inet:ipv4-prefix"] = IPv4PrefixBox.new - -function IPv4PrefixBox:set(value) - -- split the netmask and prefix up. - local raw = h.split("/", value) - local addr, err = ipv4:pton(raw[1]) - if addr == false then error(err) end - local prefix = tonumber(raw[2]) - if prefix > 32 or prefix < 0 then - error(("The prefix length is invalid. (%s)"):format(raw[2])) - end - self.value = {addr, prefix} -end -function IPv4PrefixBox.get_type() return "box-inet:ipv4-prefix" end - -local IPv6PrefixBox = {} -function IPv6PrefixBox.new(options, default) - local ret = {root={}} - local ipv6_prefix = setmetatable(ret, {__index=IPv6PrefixBox}) - if default ~= nil then - ipv6_prefix:set(default) - end - return ipv6_prefix -end -function IPv6PrefixBox:get() - return self.value -end -function IPv6PrefixBox:set(value) - -- split the netmask and prefix up. - local raw = h.split("/", value) - local addr, err = ipv6:pton(raw[1]) - if addr == false then error(err) end - local prefix = tonumber(raw[2]) - if prefix > 128 or prefix < 0 then - error(("The prefix length is invalid. (%s)"):format(raw[2])) - end - self.value = {addr, prefix} -end -function IPv6PrefixBox.get_type() return "box-inet:ipv6-prefix" end -box_types["inet:ipv6-prefix"] = IPv6PrefixBox.new - -local Enum = {} -function Enum.new(options, default) - local opts = {} - for _, option in pairs(options) do - opts[option] = option - end - local mt = {__index=Enum, options=options, default=default} - local initialized_enum = setmetatable({options=opts}, mt) - if default ~= nil then - initialized_enum:set(default) - end - return initialized_enum -end - -function Enum:get() - return self.value -end - -function Enum:set(value) - if self.options[value] then - self.value = value - else - error("Value "..value.." is not a valid option for this Enum.") - end -end -function Enum.get_type() return "box-enumeration" end -box_types["enumeration"] = Enum.new - -local Union = {} -function Union.new(types, default) - local ret = {types={}} - for _, name in pairs(types) do - -- 9.12 specifies unions cannot contain "leafref" or "empty" - if name == "empty" or name == "leafref" then - error("Union type cannot contain 'empty' or 'leafref'") - end - ret.types[name] = create_box(name, nil, default) - end - return setmetatable(ret, {__index=Union, options=types, default=default}) -end - -function Union:get() - if self.box then - return self.box:get() - end -end - -function Union:set(v) - local function setbox(b, v) b:set(v) end - for _, box in pairs(self.types) do - local valid = pcall(setbox, box, v) - if valid then - self.box = box - return -- don't want to continue. - end - end - error(("Unable to find matching type for '%s' (%s)"):format(v, type(v))) -end -function Union.get_type() return "box-union" end - -box_types["union"] = Union.new - -Container = {} -function Container.new(base, path, parent) - local ret = {root={}, base=base, path=path, parent=parent} - return setmetatable(ret, { - __newindex = function (_, k, v) - -- Validate the value prior to setting it. - local node_path = ret.path.."."..k - local schema = assert( - ret.base:get_schema(ret.path.."."..k), - ("No schema found at: %s.%s"):format(ret.path, k) - ) - - if schema.validation then - for _, validation in pairs(schema.validation) do - validation(v) - end - end - local box = assert(ret.root[k], "Unknown leaf '"..node_path.."'") - box:set(v) - end, - __index = function(t, k) - local root = rawget(t, "root") - local prop = root[k] - - if not prop then - -- This could be because it's a method defined on Container. - return rawget(Container, k) - elseif prop.get == nil then - return prop - else - return prop:get() - end - end, - }) -end - -function Container.get_type() return "container" end - -function Container:get_parent() return rawget(self, "parent") end - -function Container:set_template(template) - rawset(self, "template", template) -end -function Container:get_template() - return rawget(self, "template") -end - -local function pp(x) - if type(x) == "table" then - io.write("{") - local first = true - for k,v in pairs(x) do - if not first then - io.write(", ") - end - io.write(k.."=") - pp(v) - first = false - end - io.write("}") - elseif type(x) == "string" then - io.write(x) - else - error("Unsupported type") - end -end - - - -function Container:add_item(item) - local root = rawget(self, "root") - local base = rawget(self, "base") - local path = rawget(self, "path") - - -- Verify that the item is being added to a list type. - local schema = base:get_schema(path) - if schema:get_type() ~= "list" then - error("Can't add item to '"..schema:get_type().."'") - end - - -- Each entry will have their own container which is a copy of the template. - local template = assert(self:get_template(), "Must set template to add item") - local con = template:duplicate() - - -- Find the leaves from the schema, if it's a group then find the parent. - local leaves - if schema.uses then - leaves = base:schema_for_uses(schema).leaves - else - leaves = schema.leaves - end - - -- Create a new table of item + leavesf - local data = corelib.deepcopy(item) - for name, leaf in pairs(leaves) do - data[name] = leaf - end - - -- Add the data to the container. - for name, _ in pairs(data) do - if leaves[name] == nil then - error("Unknown field in list: '"..name.."'") - elseif item[name] == nil and leaves[name].mandatory then - error("Field "..name.." not provided but is mandatory.") - elseif item[name] then - con[name] = item[name] - end - end - - -- Create the key - local key = item[schema.key] - - if key == nil then - error("List item's key ("..schema.key..") cannot be null") - end - - -- Add the container entry to container. - root[key] = con -end - -function Container:add_container(name, container) - if container == nil then - container = Container.new(self, name) - end - self:add_to_root(name, container) -end - -function Container:add_to_root(key, value) - local root = rawget(self, "root") - root[key] = value -end - -function Container:duplicate() - local root = rawget(self, "root") - local dup = {} - - for k, v in pairs(root) do - -- If "v" is a table with the method .get_type it retrives the type - local table_type - if type(v) == "table" and v.get_type ~= nil then table_type = v.get_type() end - if table_type == "container" then - dup[k] = v:duplicate() - elseif table_type and table_type:sub(1, 4) == "box-" then - local box_type = table_type:sub(5, table_type:len()) - local options = getmetatable(v).options - local defaults = getmetatable(v).defaults - local dup_box = create_box(box_type, options, defaults) - dup[k] = dup_box - elseif type(v) == "cdata" then - local dup_ctype = ffi.new(ffi.typeof(v)) - ffi.copy(dup_ctype, v, ffi.sizeof(v)) - dup[k] = dup_ctype - end - end - - local duplicate_container = Container.new( - rawget(self, "base"), - rawget(self, "path") - ) - rawset(duplicate_container, "root", dup) - return duplicate_container -end - -function Container:get_type() return "container" end - --- Functions to help testing. -function asserterror(func, ...) - local success, val = pcall(func, ...) - if success then - error(("Asserterror failed! Returned with '%s'"):format(val)) - end -end - -function cardinality(kw, path, statements, haystack) - for s, c in pairs(statements) do - if (c[1] >= 1 and (not haystack[s])) or - (#statements[s] < c[1] and #statements[s] > c[2]) then - if c[1] == c[2] then - error(("Expected %d %s statement(s) in %s:%s"):format( - c[1], s, kw, path)) - else - local err = "Expected between %d and %d of %s statement(s) in %s:%s" - error((err):format(c[1], c[2], s, kw, path)) - end - end - end - return true -end - -function setvalue(t, k, v) - t[k] = v -end - -function selftest() - local base = require("lib.yang.yang").Base.new() - -- Register a fake schema to satisfy the requirements. - base:add_cache("mod.testbox", {}) - - local con = Container.new(base, "mod") - local root = rawget(con, "root") - - -- Test the union box (both uint8, ipv4 - valid and invalid data) - root.testbox = Union.new({"uint8", "inet:ipv4-address"}) - con.testbox = 72 - assert(con.testbox == 72) - con.testbox = "8.8.8.8" - assert(ipv4:ntop(con.testbox) == "8.8.8.8") - asserterror(setvalue, con, "testbox", "should fail") - - -- Test the IPv4 box (both valid and invalid data). - root.testbox = box_types["inet:ipv4-address"]() - con.testbox = "8.8.8.8" - assert(ipv4:ntop(con.testbox) == "8.8.8.8") - asserterror(setvalue, con, "testbox", "256.8.8.8") - - -- Test the IPv6 box (both valid and invalid data). - root.testbox = box_types["inet:ipv6-address"]() - con.testbox = "::1" - assert(ipv6:ntop(con.testbox) == "::1") - asserterror(setvalue, con, "testbox", "not ipv6") - - -- Testing enums (both valid and invalid values) - root.testbox = Enum.new({"Banana", "Apple"}) - con.testbox = "Banana" - assert(con.testbox == "Banana") - con.testbox = "Apple" - assert(con.testbox == "Apple") - asserterror(setvalue, con, "testbox", "Pear") - - -- Testing ipv4 prefix - root.testbox = IPv4PrefixBox.new() - con.testbox = "192.168.0.0/24" - local rtn = assert(con.testbox) - assert(ipv4:ntop(rtn[1]) == "192.168.0.0") - assert(rtn[2] == 24) - asserterror(setvalue, con, "testbox", "Incorrect value") - asserterror(setvalue, con, "testbox", "192.168.0.0/33") - asserterror(setvalue, con, "testbox", "192.168.0.0/-1") - asserterror(setvalue, con, "testbox", "192.256.0.0/24") - - -- Testing ipv6 prefix - root.testbox = IPv6PrefixBox.new() - con.testbox = "2001:db8::/32" - local rtn = assert(con.testbox) - assert(ipv6:ntop(rtn[1]) == "2001:db8::") - assert(rtn[2] == 32) - asserterror(setvalue, con, "testbox", "Incorrect value") - asserterror(setvalue, con, "testbox", "2001:db8::/129") - asserterror(setvalue, con, "testbox", "2001:db8::/-1") - asserterror(setvalue, con, "testbox", "FFFFF:db8::/32") - - -- Remove following when typedef is supported. - -- Test the PacketPolicy enum - root.testbox = create_box("PacketPolicy") - con.testbox = "allow" - assert(con.testbox == "allow") - con.testbox = "deny" - assert(con.testbox == "deny") - asserterror(setvalue, con, "testbox", "not allow or deny") - asserterror(setvalue, con, "testbox", "alow") -end diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index fc5973e5a4..c1a12d53b4 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -1,22 +1,8 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. --- This module implements the schema tree and validation for YANG. It represents --- the YANG statements with lua tables and provides a fast but flexible way to --- represent and validate statements. --- --- Since YANG statements are encapsulated in modules at the highest level one --- should take their pre-parsed YANG document containing the module and load it --- into the Module table. module(..., package.seeall) -local parser = require("lib.yang.parser") -local ffi = require("ffi") -ffi.cdef("long long atoll(const char *nptr);") -local function tointeger(str) - local i = ffi.C.atoll(str) - if tostring(i) == str.."LL" then - if i == tonumber(i) then return tonumber(i) else return i end - end -end +local parser = require("lib.yang.parser") +local util = require("lib.yang.util") local function error_with_path(path, msg, ...) error(string.format("%s: "..msg, path, ...)) @@ -69,12 +55,8 @@ local function parse_range(node, range) local function parse_part(part) local l, r = part:match("^%s*([^%.]*)%s*%.%.%s*([^%s]*)%s*$") assert_with_path(l, node.path, 'bad range component: %s', part) - if l ~= 'min' then - l = assert_with_path(tointeger(l), node.path, "bad integer: %s", l) - end - if r ~= 'max' then - r = assert_with_path(tointeger(r), node.path, "bad integer: %s", r) - end + if l ~= 'min' then l = util.tointeger(l) end + if r ~= 'max' then r = util.tointeger(r) end return { l, r } end local parts = range:split("|") diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua new file mode 100644 index 0000000000..8358388f0a --- /dev/null +++ b/src/lib/yang/util.lua @@ -0,0 +1,58 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local ffi = require("ffi") + +-- Parse inet:mac-address using ethernet:pton +-- Parse inet:ipv4-address using ipv4:pton +-- Parse inet:ipv6-address using ipv6:pton + +-- Parse inet:ipv4-prefix? +-- Parse inet:ipv6-prefix? + +ffi.cdef([[ +unsigned long long strtoull (const char *nptr, const char **endptr, int base); +]]) + +function tointeger(str, what, min, max) + if not what then what = 'integer' end + local str = assert(str, 'missing value for '..what) + local start = 1 + local is_negative + local base = 10 + if str:match('^-') then start, is_negative = 2, true + elseif str:match('^+') then start = 2 end + if str:match('^0x', start) then base, start = 16, start + 2 + elseif str:match('^0', start) then base = 8 end + str = str:lower() + local function check(test) + return assert(test, 'invalid numeric value for '..what..': '..str) + end + check(start <= str:len()) + -- FIXME: check that res did not overflow the 64-bit number + local res = ffi.C.strtoull(str:sub(start), nil, base) + if is_negative then + res = ffi.new('int64_t[1]', -1*res)[0] + check(res <= 0) + end + if min then check(min <= res) end + if max then check(res <= max) end + -- Only return Lua numbers for values within int32 + uint32 range. + if -0x8000000 <= res and res <= 0xffffffff then return tonumber(res) end + return res +end + +function selftest() + assert(tointeger('0') == 0) + assert(tointeger('-0') == 0) + assert(tointeger('10') == 10) + assert(tointeger('-10') == -10) + assert(tointeger('010') == 8) + assert(tointeger('-010') == -8) + assert(tointeger('0xffffffff') == 0xffffffff) + assert(tointeger('0xffffffffffffffff') == 0xffffffffffffffffULL) + assert(tointeger('0x7fffffffffffffff') == 0x7fffffffffffffffULL) + assert(tointeger('0xffffffffffffffff') == 0xffffffffffffffffULL) + assert(tointeger('-0x7fffffffffffffff') == -0x7fffffffffffffffLL) + assert(tointeger('-0x8000000000000000') == -0x8000000000000000LL) +end From 0934afb3f2f0c887ad351db376da55b9525130af Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 15:55:01 +0100 Subject: [PATCH 080/631] yang: Primitive types for ipv4 address etc Arrange to frob well-known typedefs into being "primitive". The data parser will arrange to handle them appropriately. --- src/lib/yang/data.lua | 4 +--- src/lib/yang/schema.lua | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 42eff71a60..3c61f2e1a9 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -112,9 +112,7 @@ local function bit_validator(range, f) return f end local function enum_validator(range, f) return f end local function value_parser(typ) - local prim = typ.base_type - -- FIXME: perhaps cache the primitive type on all type nodes. - while type(prim) ~= 'string' do prim = prim.base_type end + local prim = typ.primitive_type local parse = assert(primitive_parsers[prim], prim) local function validate(val) end validate = range_validator(typ.range, validate) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index c1a12d53b4..c773ae1d24 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -653,15 +653,18 @@ function resolve(schema, features) node = shallow_copy(node) local success, typedef = pcall(lookup, env, 'typedefs', node.id) if success then - -- Could be that typedef is still lazy. We didn't use + -- Typedefs are lazy, so force their value. We didn't use -- lookup_lazy because we don't want the pcall to hide errors -- from the lazy expansion. - if type(typedef) == 'function' then typedef = typedef() end + typedef = typedef() node.base_type = typedef + if not node.primitive_type then + node.primitive_type = assert(typedef.type.primitive_type) + end else -- If the type name wasn't bound, it must be primitive. assert(primitive_types[node.id], 'unknown type: '..node.id) - node.base_type = node.id + node.primitive_type = node.id end return node end @@ -772,6 +775,20 @@ function resolve(schema, features) submodules=pop_prop(schema, 'submodules')}) end +local primitive_types = { + ['ietf-inet-types']=set('ipv4-address', 'ipv6-address', + 'ipv4-prefix', 'ipv6-prefix'), + ['ietf-yang-types']=set('mac-address') +} + +-- NB: mutates schema in place! +local function primitivize(schema) + for k, _ in pairs(primitive_types[schema.id] or {}) do + assert(schema.typedefs[k]).primitive_type = k + end + return schema +end + function parse_schema(src, filename) return schema_from_ast(parser.parse_string(src, filename)) end @@ -780,10 +797,10 @@ function parse_schema_file(filename) end function load_schema(src, filename) - return resolve(strip(parse_schema(src, filename))) + return resolve(primitivize(strip(parse_schema(src, filename)))) end function load_schema_file(filename) - return resolve(strip(parse_schema_file(filename))) + return resolve(primitivize(strip(parse_schema_file(filename)))) end function load_schema_by_name(name, revision) -- FIXME: @ is not valid in a Lua module name. From 103b7f3257ba1d3532ce640212a90722cb63736b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 15:59:06 +0100 Subject: [PATCH 081/631] Fix failing lib.yang.parser test --- src/lib/yang/parser.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 43ee870c11..7852513ef7 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -355,5 +355,7 @@ function selftest() test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type"}}}}) - parse_file(require('lib.yang.example_yang')) + parse_string(require('lib.yang.ietf_inet_types_yang')) + parse_string(require('lib.yang.ietf_yang_types_yang')) + parse_string(require('lib.yang.ietf_softwire_yang')) end From 8c8dc727b0ded04be1899412c766bebd9530345d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 31 Oct 2016 16:40:45 +0100 Subject: [PATCH 082/631] Arrange to parse ipv4-address etc as primitive types Also add lib.yang.value module. --- src/lib/yang/data.lua | 59 ++++---------------- src/lib/yang/schema.lua | 11 ++-- src/lib/yang/value.lua | 116 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 54 deletions(-) create mode 100644 src/lib/yang/value.lua diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 3c61f2e1a9..0812c3ce24 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -5,6 +5,7 @@ module(..., package.seeall) local parse_string = require("lib.yang.parser").parse_string local schema = require("lib.yang.schema") local util = require("lib.yang.util") +local value = require("lib.yang.value") -- FIXME: -- Parse inet:mac-address using ethernet:pton @@ -55,55 +56,6 @@ local function integer_type(min, max) end end -local primitive_parsers = {} - -primitive_parsers.int8 = integer_type(-0xf0, 0x7f) -primitive_parsers.int16 = integer_type(-0xf000, 0x7fff) -primitive_parsers.int32 = integer_type(-0xf000000, 0x7fffffff) -primitive_parsers.int64 = integer_type(-0xf00000000000000LL, 0x7fffffffffffffffLL) -primitive_parsers.uint8 = integer_type(0, 0xff) -primitive_parsers.uint16 = integer_type(0, 0xffff) -primitive_parsers.uint32 = integer_type(0, 0xffffffff) -primitive_parsers.uint64 = integer_type(0, 0xffffffffffffffffULL) -function primitive_parsers.binary(str, k) - error('unimplemented: binary') - return str -end -function primitive_parsers.bits(str, k) - error('unimplemented: bits') -end -function primitive_parsers.boolean(str, k) - local str = assert(str, 'missing value for '..k) - if str == 'true' then return true end - if str == 'false' then return false end - error('bad boolean value: '..str) -end -function primitive_parsers.decimal64(str, k) - error('unimplemented: decimal64') -end -function primitive_parsers.empty(str, k) - assert(not str, 'unexpected value for '..k) - return true -end -function primitive_parsers.enumeration(str, k) - return assert(str, 'missing value for '..k) -end -function primitive_parsers.identityref(str, k) - error('unimplemented: identityref') -end -primitive_parsers['instance-identifier'] = function(str, k) - error('unimplemented: instance-identifier') -end -function primitive_parsers.leafref(str, k) - error('unimplemented: leafref') -end -function primitive_parsers.string(str, k) - return assert(str, 'missing value for '..k) -end -function primitive_parsers.union(str, k) - error('unimplemented: union') -end - -- FIXME :) local function range_validator(range, f) return f end local function length_validator(range, f) return f end @@ -113,7 +65,7 @@ local function enum_validator(range, f) return f end local function value_parser(typ) local prim = typ.primitive_type - local parse = assert(primitive_parsers[prim], prim) + local parse = assert(value.types[prim], prim).parse local function validate(val) end validate = range_validator(typ.range, validate) validate = length_validator(typ.length, validate) @@ -285,6 +237,7 @@ function selftest() local test_schema = schema.load_schema([[module fruit { namespace "urn:testing:fruit"; prefix "fruit"; + import ietf-inet-types {prefix inet; } grouping fruit { leaf name { type string; @@ -302,6 +255,10 @@ function selftest() leaf description { type string; } list contents { uses fruit; key name; } } + leaf addr { + description "internet of fruit"; + type inet:ipv4-address; + } }]]) local data = load_data_for_schema(test_schema, [[ @@ -311,6 +268,7 @@ function selftest() contents { name bar; score 8; } contents { name baz; score 9; tree-grown true; } } + addr 1.2.3.4; ]]) assert(data['fruit-bowl'].description == 'ohai') assert(data['fruit-bowl'].contents:lookup('foo').name == 'foo') @@ -322,4 +280,5 @@ function selftest() assert(data['fruit-bowl'].contents:lookup('baz').name == 'baz') assert(data['fruit-bowl'].contents:lookup('baz').score == 9) assert(data['fruit-bowl'].contents:lookup('baz')['tree-grown'] == true) + assert(require('lib.protocol.ipv4'):ntop(data.addr) == '1.2.3.4') end diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index c773ae1d24..651d62aaa3 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -658,9 +658,7 @@ function resolve(schema, features) -- from the lazy expansion. typedef = typedef() node.base_type = typedef - if not node.primitive_type then - node.primitive_type = assert(typedef.type.primitive_type) - end + node.primitive_type = assert(typedef.primitive_type) else -- If the type name wasn't bound, it must be primitive. assert(primitive_types[node.id], 'unknown type: '..node.id) @@ -696,7 +694,12 @@ function resolve(schema, features) return nil, env end end - if node.type then node.type = visit_type(node.type, env) end + if node.type then + node.type = visit_type(node.type, env) + if not node.primitive_type then + node.primitive_type = node.type.primitive_type + end + end for k,v in pairs(node.body or {}) do if v.kind == 'uses' then -- Inline "grouping" into "uses". diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua new file mode 100644 index 0000000000..539d8f36bd --- /dev/null +++ b/src/lib/yang/value.lua @@ -0,0 +1,116 @@ +-- Use of this source code is governed by the Apache 2.0 license; see +-- COPYING. +module(..., package.seeall) + +local util = require("lib.yang.util") +local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") +local ethernet = require("lib.protocol.ethernet") + +-- FIXME: +-- Parse inet:mac-address using ethernet:pton +-- Parse inet:ipv4-address using ipv4:pton +-- Parse inet:ipv6-address using ipv6:pton +-- Parse inet:ipv4-prefix? +-- Parse inet:ipv6-prefix? + +types = {} + +local function integer_type(min, max) + local ret = {} + function ret.parse(str, what) + return util.tointeger(str, what, min, max) + end + function ret.tostring(val) + local str = tostring(val) + if str:match("ULL") then return str:sub(1, -4) + elseif str:match("LL") then return str:sub(1, -3) + else return str end + end + return ret +end + +types.int8 = integer_type(-0xf0, 0x7f) +types.int16 = integer_type(-0xf000, 0x7fff) +types.int32 = integer_type(-0xf000000, 0x7fffffff) +types.int64 = integer_type(-0xf00000000000000LL, 0x7fffffffffffffffLL) +types.uint8 = integer_type(0, 0xff) +types.uint16 = integer_type(0, 0xffff) +types.uint32 = integer_type(0, 0xffffffff) +types.uint64 = integer_type(0, 0xffffffffffffffffULL) + +local function unimplemented(type_name) + local ret = {} + function ret.parse(str, what) + error('unimplemented '..type_name..' when parsing '..what) + end + function ret.tostring(val) + return tostring(val) + end + return ret +end + +types.binary = unimplemented('binary') +types.bits = unimplemented('bits') + +types.boolean = {} +function types.boolean.parse(str, what) + local str = assert(str, 'missing value for '..what) + if str == 'true' then return true end + if str == 'false' then return false end + error('bad boolean value: '..str) +end +function types.boolean.tostring(val) + return tostring(val) +end + +types.decimal64 = unimplemented('decimal64') +types.empty = unimplemented('empty') +types.identityref = unimplemented('identityref') +types['instance-identifier'] = unimplemented('instance-identifier') +leafref = unimplemented('leafref') + +types.string = {} +function types.string.parse(str, what) + return assert(str, 'missing value for '..what) +end +function types.string.tostring(val) + return val +end + +types.union = unimplemented('union') + +types['ipv4-address'] = { + parse = function(str, what) return assert(ipv4:pton(str)) end, + tostring = function(val) return ipv4:ntop(val) end +} + +types['ipv6-address'] = { + parse = function(str, what) return assert(ipv6:pton(str)) end, + tostring = function(val) return ipv6:ntop(val) end +} + +types['mac-address'] = { + parse = function(str, what) return assert(ethernet:pton(str)) end, + tostring = function(val) return ethernet:ntop(val) end +} + +types['ipv4-prefix'] = { + parse = function(str, what) + local prefix, len = str:match('^([^/]+)/(.*)$') + return { assert(ipv4:pton(prefix)), util.tointeger(len, 1, 32) } + end, + tostring = function(val) return ipv4:ntop(val[1])..'/'..tostring(val[2]) end +} + +types['ipv6-prefix'] = { + parse = function(str, what) + local prefix, len = str:match('^([^/]+)/(.*)$') + return { assert(ipv6:pton(prefix)), util.tointeger(len, 1, 128) } + end, + tostring = function(val) return ipv6:ntop(val[1])..'/'..tostring(val[2]) end +} + +function selftest() + assert(types['uint8'].parse('100') == 100) +end From f803b8968728d6ca6be236a678d05f6dc6490568 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Nov 2016 11:04:53 +0100 Subject: [PATCH 083/631] Finish lib.yang README --- src/lib/yang/README.md | 124 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 7 deletions(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index 66a8b28181..ce7c2eabdc 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -144,19 +144,129 @@ hand. #### Compiled configurations +[TODO] We will support compiling configurations to an efficient binary +representation that can be loaded without validation. + #### Querying and updating configurations +[TODO] We will need to be able to serialize a configuration back to +source, for when a user asks what the configuration of a device is. We +will also need to serialize partial configurations, for when the user +asks for just a part of the configuration. + +[TODO] We will need to support updating the configuration of a running +snabb application. We plan to compile the candidate configuration in a +non-worker process, then signal the worker to reload its configuration. + +[TODO] We will need to support incremental configuration updates, for +example to add or remove a binding table entry for the lwAFTR. In this +way we can avoid a full reload of the configuration, minimizing packet +loss. + #### State data -#### Function reference +[TODO] We need to map the state data exported by a Snabb process +(counters, etc) to YANG-format data. Perhaps this can be done in a +similar way as configuration compilation: the configuration facility in +the Snabb binary compiles a YANG state data file and periodically +updates it by sampling the data plane, and then we re-use the +configuration serialization facilities to serialize (potentially +partial) state data. + +#### API reference + +The public entry point to the YANG library is the `lib.yang.yang` +module, which exports the following bindings: + +— Function **load_schema** *src* *filename* + +Load a YANG schema from the string *src*. *filename* is an optional +file name for use in error messages. Returns a YANG schema object. + +Schema objects do have useful internal structure but they are not part +of the documented interface. + +— Function **load_schema_file** *filename* + +Load a YANG schema from the file named *filename*. Returns a YANG +schema object. + +— Function **load_schema_by_name** *name* *revision* + +Load the given named YANG schema. The *name* indicates the canonical +name of the schema, which appears as `module *name* { ... }` in the YANG +schema itself, or as `import *name* { ... }` in other YANG modules that +import this module. *revision* optionally indicates that a certain +revision data should be required. + +— Function **load_data_for_schema** *schema* *src* *filename* + +Given the schema object *schema*, load the configuration from the string +*src*. Returns a parsed configuration as a plain old Lua value that +tries to represent configuration values using appropriate Lua types. + +The top-level result from parsing will be a table whose keys are the +top-level configuration options. For example in the above example: + +``` +active true; +routes { + route { addr 1.2.3.4; port 1; } + route { addr 2.3.4.5; port 10; } + route { addr 3.4.5.6; port 2; } +} +``` + +In this case, the result would be a table with two keys, `active` and +`routes`. The value of the `active` key would be Lua boolean `true`. + +The `routes` container is just another table of the same kind. +(Remember however that only containers with `presence true;` have +corresponding nodes in the configuration syntax, and corresponding +sub-tables in the result configuration objects.) + +Inside the `routes` container is the `route` list, which is also +represented as a table. Recall that in YANG, `list` types are really +key-value associations, so the `route` table has a `:lookup` method to +get its sub-items. Therefore to get the port for address 1.2.3.4, you +would do: + +```lua +local yang = require('lib.yang.yang') +local ipv4 = require('lib.protocol.ipv4) +local data = yang.load_data_for_schema(router_schema, conf_str) +local port = data.routes.route:lookup(ipv4:pton('1.2.3.4')).port +assert(port == 1) +``` + +Here we see that integer values like the `port` leaves are represented +directly as Lua numbers, if they fit within the `uint32` or `int32` +range. Integers outside that range are represented as `uint64_t` if +they are positive, or `int64_t` otherwise. + +Boolean values are represented using normal Lua booleans, of course. + +String values are just parsed to Lua strings, with the normal Lua +limitation that UTF-8 data is not decoded. Lua strings look like +strings but really they are byte arrays. -All of these functions are on modules in the `lib.yang` path. For -example, to have access to: +There is special support for the `ipv4-address`, `ipv4-prefix`, +`ipv6-address`, and `ipv6-address` types from `ietf-inet-types`, and +`mac-address` from `ietf-yang-types`. Values of these types are instead +parsed to raw binary data that is compatible with the relevant parts of +Snabb's `lib.protocol` facility. -— Function **schema.load_schema_by_name** *name* +Returning to compound configuration data types, configuration for +`leaf-list` schema nodes are represented as normal arrays, whose values +are instances of the leaf types. -Then do `local schema = require('lib.yang.schema')`. +Note that there are a number of value types that are not implemented, +including some important ones like `union`, and the `list` type +representation needs further optimization. We aim to compile `list` +values directly to `ctable` instances where possible. Patches are +welcome :) -— Function **schema.load_schema_by_name** *name* +— Function **load_data_for_schema_by_name** *schema_name* *name* *filename* -Load up schema by name. [TODO write more here.] +Like `load_data_for_schema`, but identifying the schema by name instead +of by value, as in `load_schema_by_name`. From c04ab993ba35f47c0c6eeacdc2b5509313949903 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Nov 2016 11:34:43 +0100 Subject: [PATCH 084/631] lib.yang.util.tointeger allows for negative int64 min values --- src/lib/yang/util.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 8358388f0a..61391c307e 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -34,8 +34,13 @@ function tointeger(str, what, min, max) if is_negative then res = ffi.new('int64_t[1]', -1*res)[0] check(res <= 0) + if min then check(min <= 0 and min <= res) end + else + -- Only compare min and res if both are positive, otherwise if min + -- is a negative int64_t then the comparison will treat it as a + -- large uint64_t. + if min then check(min <= 0 or min <= res) end end - if min then check(min <= res) end if max then check(res <= max) end -- Only return Lua numbers for values within int32 + uint32 range. if -0x8000000 <= res and res <= 0xffffffff then return tonumber(res) end From dfd16f7e2133ed69947844fabd8d9528a5af22a1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Nov 2016 11:42:31 +0100 Subject: [PATCH 085/631] Yang primitive value types record their ctypes --- src/lib/yang/value.lua | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua index 539d8f36bd..cf7787aa5a 100644 --- a/src/lib/yang/value.lua +++ b/src/lib/yang/value.lua @@ -5,6 +5,8 @@ module(..., package.seeall) local util = require("lib.yang.util") local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") +local ffi = require("ffi") +local bit = require("bit") local ethernet = require("lib.protocol.ethernet") -- FIXME: @@ -16,8 +18,15 @@ local ethernet = require("lib.protocol.ethernet") types = {} -local function integer_type(min, max) - local ret = {} +local function integer_type(ctype) + local ret = {ctype=ctype} + local min, max = ffi.new(ctype, 0), ffi.new(ctype, -1) + if max < 0 then + -- A signed type. Hackily rely on unsigned types having 'u' + -- prefix. + max = ffi.new(ctype, bit.rshift(ffi.new('u'..ctype, max), 1)) + min = max - max - max - 1 + end function ret.parse(str, what) return util.tointeger(str, what, min, max) end @@ -30,14 +39,14 @@ local function integer_type(min, max) return ret end -types.int8 = integer_type(-0xf0, 0x7f) -types.int16 = integer_type(-0xf000, 0x7fff) -types.int32 = integer_type(-0xf000000, 0x7fffffff) -types.int64 = integer_type(-0xf00000000000000LL, 0x7fffffffffffffffLL) -types.uint8 = integer_type(0, 0xff) -types.uint16 = integer_type(0, 0xffff) -types.uint32 = integer_type(0, 0xffffffff) -types.uint64 = integer_type(0, 0xffffffffffffffffULL) +types.int8 = integer_type('int8_t') +types.int16 = integer_type('int16_t') +types.int32 = integer_type('int32_t') +types.int64 = integer_type('int64_t') +types.uint8 = integer_type('uint8_t') +types.uint16 = integer_type('uint16_t') +types.uint32 = integer_type('uint32_t') +types.uint64 = integer_type('uint64_t') local function unimplemented(type_name) local ret = {} @@ -81,21 +90,25 @@ end types.union = unimplemented('union') types['ipv4-address'] = { + ctype = 'uint8_t[4]', parse = function(str, what) return assert(ipv4:pton(str)) end, tostring = function(val) return ipv4:ntop(val) end } types['ipv6-address'] = { + ctype = 'uint8_t[16]', parse = function(str, what) return assert(ipv6:pton(str)) end, tostring = function(val) return ipv6:ntop(val) end } types['mac-address'] = { + ctype = 'uint8_t[6]', parse = function(str, what) return assert(ethernet:pton(str)) end, tostring = function(val) return ethernet:ntop(val) end } types['ipv4-prefix'] = { + ctype = 'struct { uint8_t[4] prefix; uint8_t len; }', parse = function(str, what) local prefix, len = str:match('^([^/]+)/(.*)$') return { assert(ipv4:pton(prefix)), util.tointeger(len, 1, 32) } @@ -104,6 +117,7 @@ types['ipv4-prefix'] = { } types['ipv6-prefix'] = { + ctype = 'struct { uint8_t[16] prefix; uint8_t len; }', parse = function(str, what) local prefix, len = str:match('^([^/]+)/(.*)$') return { assert(ipv6:pton(prefix)), util.tointeger(len, 1, 128) } @@ -112,5 +126,11 @@ types['ipv6-prefix'] = { } function selftest() - assert(types['uint8'].parse('100') == 100) + assert(types['uint8'].parse('0') == 0) + assert(types['uint8'].parse('255') == 255) + assert(not pcall(types['uint8'].parse, '256')) + assert(types['int8'].parse('-128') == -128) + assert(types['int8'].parse('0') == 0) + assert(types['int8'].parse('127') == 127) + assert(not pcall(types['int8'].parse, '128')) end From 40100612edc9e7bbc6844d1ad97cf7edc85a1a10 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 1 Nov 2016 13:29:33 +0100 Subject: [PATCH 086/631] Add backlink for named programs --- src/core/app.lua | 32 +++++++++++++++++++++++++++----- src/core/main.lua | 17 ++++++++--------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 22ef8b42a0..e16fc4f515 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -134,20 +134,35 @@ end -- an active process the function will error displaying an appropriate error message. -- Any other process can enumerate programs by their name and PID using enumerate_named_programs. function claim_name(name) + function name_from_path(dir) + -- Extracts last file/folder name from path + -- e.g. "/var/run/snabb/hello" -> "hello" + local ds, de = string.find(dir, "/[^/]*$") + return string.sub(dir, ds+1, de) + end + configuration[name] = name local namedir = "by-name/" .. name local namedirqualified = named_program_root .. "/" .. name local piddir = shm.root .. "/" .. S.getpid() + local backlinkdir = piddir.."/name-backref" -- Verify that the by-name directory exists. shm.mkdir(namedir) - + + -- Verify that we've not already claimed a name + if S.lstat(backlinkdir) then + -- Read name of program for error message. + local fnamedir = S.readlink(backlinkdir) + local fname = name_from_path(fnamedir) + error("Program already claimed name '"..fname.."', cannot claim name '"..name.."'") + end + -- Check if a symlink to a named program with the same name exists, if not, unlink it. if S.lstat(namedirqualified) then -- It exists, lets check what the PID is. local piddir = S.readlink(namedirqualified) - local ps, pe = string.find(piddir, "/[^/]*$") - local pid = tonumber(string.sub(piddir, ps+1, pe)) + local pid = tonumber(name_from_path(piddir)) if S.kill(pid, 0) then -- It's a running process, we should display an error message. error("Name has been claimed by a currently running process: "..namedirqualified) @@ -162,6 +177,12 @@ function claim_name(name) if rtn == -1 then error("Error creating program name symlink in: "..namedirqualified) end + + -- Create a backlink so to the symlink so we can easily cleanup + local rtn, err = S.symlink(namedirqualified, backlinkdir) + if rtn == -1 then + error("Error creating backlink for: "..namedirqualified) + end end -- Enumerates all the named programs with their PID @@ -595,13 +616,14 @@ function selftest () -- Test claiming and enumerating app names local basename = "testapp" claim_name(basename.."1") - claim_name(basename.."2") + + -- Ensure to claim two names fails + assert(not pcall(claim_name, basename.."2")) -- Lets check if they can be enumerated. local progs = enumerate_named_programs() assert(progs) assert(progs["testapp1"]) - assert(progs["testapp2"]) -- Ensure that trying to take the same name fails assert(not pcall(claim_name, basename.."1")) diff --git a/src/core/main.lua b/src/core/main.lua index e03a156c18..7b20a69019 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -143,16 +143,15 @@ end -- Cleanup after Snabb process. function shutdown (pid) if not _G.developer_debug and not lib.getenv("SNABB_SHM_KEEP") then - shm.unlink("/"..pid) - - -- Look through the named apps and unlink any which are for this process. - local npid = tonumber(pid) - local progs = app.enumerate_named_programs(true) - for name, p in pairs(progs) do - if p == npid then - S.unlink(app.named_program_root .. "/" .. name) - end + -- Check if there is a backlink to the named app, if so cleanup that. + local backlink = shm.root.."/"..pid.."/name-backref" + if S.lstat(backlink) then + local name_link = S.readlink(backlink) + S.unlink(name_link) + S.unlink(backlink) end + + shm.unlink("/"..pid) end end From caa3689319e4860cfb61c3b3618f9bb56e0197da Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Nov 2016 16:43:09 +0100 Subject: [PATCH 087/631] YANG data grammar separates keys, values for tables A "table" production was previously represented as having a flat value like a struct, with some of the members marked as keys. Instead separate the key from the value, in anticipation of making better associative maps, and instead of providing "lookup" on the data, provide "get_entry", "get_key", and "get_value", as ctable does. --- src/lib/yang/data.lua | 83 ++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 0812c3ce24..8f96ab8d65 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -39,9 +39,13 @@ function data_grammar_from_schema(schema) return {[node.id]={type='array', element_type=node.type}} end function handlers.list(node) - local keys = {} - for key in node.key:split(' +') do table.insert(keys, key) end - return {[node.id]={type='table', keys=keys, members=visit_body(node)}} + local members=visit_body(node) + local keys, values = {}, {} + for k in node.key:split(' +') do keys[k] = assert(members[k]) end + for k,v in pairs(members) do + if not keys[k] then values[k] = v end + end + return {[node.id]={type='table', keys=keys, values=values}} end function handlers.leaf(node) return {[node.id]={type='scalar', argument_type=node.type, @@ -158,31 +162,50 @@ local function scalar_parser(keyword, argument_type, default, mandatory) return {init=init, parse=parse, finish=finish} end -local function table_parser(keyword, keys, members) - -- This is a temporary lookup until we get the various Table kinds - -- working. - local function lookup(out, k) - for _,v in ipairs(out) do - if #keys == 1 then - if v[keys[1]] == k then return v end +-- Simple temporary associative array until we get the various Table +-- kinds working. +local function make_assoc() + local assoc = {} + function assoc:get_entry(k) + assert(type(k) ~= 'table', 'multi-key lookup unimplemented') + for _,entry in ipairs(self) do + for _,v in pairs(entry.key) do + if v == k then return entry end end end error('not found: '..k) end - local function init() return {lookup=lookup} end + function assoc:get_key(k) return self:get_entry(k).key end + function assoc:get_value(k) return self:get_entry(k).value end + function assoc:add(k, v, check) + if check then assert(not self:get_entry(k)) end + table.insert(self, {key=k, value=v}) + end + return assoc +end + +local function table_parser(keyword, keys, values) + local members = {} + for k,v in pairs(keys) do members[k] = v end + for k,v in pairs(values) do members[k] = v end local parser = struct_parser(keyword, members) + + local function init() return make_assoc() end local function parse1(node) assert_compound(node, keyword) return parser.finish(parser.parse(node, parser.init())) end - local function parse(node, out) - -- TODO: tease out key from value, add to associative array - table.insert(out, parse1(node)) - return out + local function parse(node, assoc) + local struct = parse1(node) + local key, value = {}, {} + for k,v in pairs(struct) do + if keys[k] then key[k] = v else value[k] = v end + end + assoc:add(key, value) + return assoc end - local function finish(out) - -- FIXME check min-elements - return out + local function finish(assoc) + return assoc end return {init=init, parse=parse, finish=finish} end @@ -210,7 +233,8 @@ function data_parser_from_grammar(production) return array_parser(keyword, production.element_type) end function handlers.table(keyword, production) - return table_parser(keyword, production.keys, visitn(production.members)) + return table_parser(keyword, visitn(production.keys), + visitn(production.values)) end function handlers.scalar(keyword, production) return scalar_parser(keyword, production.argument_type, @@ -271,14 +295,17 @@ function selftest() addr 1.2.3.4; ]]) assert(data['fruit-bowl'].description == 'ohai') - assert(data['fruit-bowl'].contents:lookup('foo').name == 'foo') - assert(data['fruit-bowl'].contents:lookup('foo').score == 7) - assert(data['fruit-bowl'].contents:lookup('foo')['tree-grown'] == nil) - assert(data['fruit-bowl'].contents:lookup('bar').name == 'bar') - assert(data['fruit-bowl'].contents:lookup('bar').score == 8) - assert(data['fruit-bowl'].contents:lookup('bar')['tree-grown'] == nil) - assert(data['fruit-bowl'].contents:lookup('baz').name == 'baz') - assert(data['fruit-bowl'].contents:lookup('baz').score == 9) - assert(data['fruit-bowl'].contents:lookup('baz')['tree-grown'] == true) + local contents = data['fruit-bowl'].contents + assert(contents:get_entry('foo').key.name == 'foo') + assert(contents:get_entry('foo').value.score == 7) + assert(contents:get_key('foo').name == 'foo') + assert(contents:get_value('foo').score == 7) + assert(contents:get_value('foo')['tree-grown'] == nil) + assert(contents:get_key('bar').name == 'bar') + assert(contents:get_value('bar').score == 8) + assert(contents:get_value('bar')['tree-grown'] == nil) + assert(contents:get_key('baz').name == 'baz') + assert(contents:get_value('baz').score == 9) + assert(contents:get_value('baz')['tree-grown'] == true) assert(require('lib.protocol.ipv4'):ntop(data.addr) == '1.2.3.4') end From 2ece77a33f7c5037f0f387364e40f7dd9b796657 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Nov 2016 17:46:35 +0100 Subject: [PATCH 088/631] Add binary compilation for YANG configuration data Still to do are the raw containers, and to write the loader. --- src/lib/yang/binary.lua | 296 ++++++++++++++++++++++++++++++++++++++++ src/lib/yang/stream.lua | 167 +++++++++++++++++++++++ src/lib/yang/value.lua | 2 +- 3 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 src/lib/yang/binary.lua create mode 100644 src/lib/yang/stream.lua diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua new file mode 100644 index 0000000000..a5b30a4458 --- /dev/null +++ b/src/lib/yang/binary.lua @@ -0,0 +1,296 @@ +-- Use of this source code is governed by the Apache 2.0 license; see +-- COPYING. +module(..., package.seeall) + +local ffi = require("ffi") +local schema = require("lib.yang.schema") +local util = require("lib.yang.util") +local value = require("lib.yang.value") +local stream = require("lib.yang.stream") +local data = require('lib.yang.data') + +local MAGIC = "yangconf" +local VERSION = 0x00001000 + +local header_t = ffi.typeof([[ +struct { + uint8_t magic[8]; + uint32_t version; + uint64_t source_mtime_sec; + uint32_t source_mtime_nsec; + uint32_t schema_name; + uint32_t revision_date; + uint32_t data_start; + uint32_t data_len; + uint32_t strtab_start; + uint32_t strtab_len; +} +]]) + +-- A string table is written out as a uint32 count, followed by that +-- many offsets indicating where the Nth string ends, followed by the +-- string data for those strings. +local function string_table_builder() + local strtab = {} + local strings = {} + local count = 0 + function strtab:intern(str) + if strings[str] then return strings[str] end + strings[str] = count + count = count + 1 + return strings[str] + end + function strtab:emit(stream) + local by_index = {} + for str, idx in pairs(strings) do by_index[idx] = str end + stream:align(4) + local strtab_start = stream.written + stream:write_uint32(count) + local str_end = 0 + for i=0,count-1 do + str_end = str_end + by_index[i]:len() + stream:write_uint32(str_end) + end + for i=0,count-1 do + str_end = str_end + by_index[i]:len() + stream:write(by_index[i], by_index[i]:len()) + end + return strtab_start, stream.written - strtab_start + end + return strtab +end + +local function read_string_table(stream, strtab_len) + stream:align(4) + assert(strtab_len >= 4) + local count = stream:read_uint32() + assert(strtab_len >= (4 * (count + 1))) + local offsets = stream:read_array(ffi.typeof('uint32_t'), count) + assert(strtab_len == (4 * (count + 1)) + offsets[count-1]) + local strings = {} + local offset = 0 + for i=0,count-1 do + local len = offsets[i] - offset + assert(len >= 0) + strings[i] = stream:read_string(len) + offset = offset + len + end + return strings +end + +-- TODO: Implement raw structs, arrays, and tables, as follows: + +-- RawStruct ::= Tag CData +-- CData ::= CType CDataContents +-- CType ::= StringRef of ffi type +-- CDataContents ::= raw data as bytes +-- +-- -- CType is something like "struct { int32 a; double b; }", suitable +-- -- to pass to ffi.typeof. The data itself will be aligned to the type +-- -- alignment and padded on the end to next 32-bit boundary. + +-- -- For RawArray, the CType indicates the type of a single item. +-- RawArray ::= Tag ArrayCount CData +-- CArray ::= CType CArrayContents +-- CType ::= StringRef of ffi type +-- CArrayContents ::= raw data as bytes + +-- -- Here the raw keys or values, respectively, are collected into an +-- -- array as in RawArray, and then the tagged values or keys follow. +-- -- Note that the CType denotes the key or value type, not the type of +-- -- the array as a whole. As before, when the table is loaded by the +-- -- data plane, it will have to build a Lua table on its own. (Perhaps +-- -- we could optimize this in the RawTagged case by using a ctable for +-- -- the keys.) +-- RawTaggedTable ::= Tag TableCount CData SubData... +-- TaggedRawTable ::= Tag TableCount CData SubData... +-- +-- -- This is the case we are optimizing for with the binding table: here +-- -- we just serialize the CTable. +-- RawRawTable ::= Tag CTable +-- CTable ::= KeyType ValueType CTableHeader CTableData +-- KeyType ::= CType +-- ValueType ::= CType +-- CTableHeader ::= CTableSize CTableOccupancy CTableMaxDisplacement \ +-- CTableMaxOccupancyRate CTableMinOccupancyRate +-- CTableSize ::= uint32 count for number of slots in table including empty +-- CTableOccupancy ::= uint32 count for number of occupied slots in table +-- CTableMaxDisplacement ::= uint32 measured maximum displacement for table +-- CTableMaxOccupancyRate ::= configured maximum fraction of occupied slots as double before resize +-- CTableMinOccupancyRate ::= configured minimum fraction of occupied slots as double before resize +-- CTableData ::= raw entries as bytes +-- +-- -- The number of slots in the table will be CTableSize + +-- -- CTableMaxDisplacement + 1. We assume that the code that generates +-- -- the compiled configuration is using the same hash function as the +-- -- code that uses the compiled table. + +local value_emitters = {} +local function value_emitter(primitive_type) + print(primitive_type) + local ctype = assert(assert(value.types[primitive_type]).ctype) + if value_emitters[ctype] then return value_emitters[ctype] end + local type = ffi.typeof(ctype) + local align = ffi.alignof(type) + local size = ffi.sizeof(type) + local buf = ffi.typeof('$[1]', type)() + local function emit(val, stream) + buf[0] = val + stream:write_ptr(buf) + end + value_emitters[ctype] = emit + return emit +end + +function table_size(tab) + local size = 0 + for k,v in pairs(tab) do size = size + 1 end + return size +end + +function data_emitter(production) + local handlers = {} + local function visit1(production) + return assert(handlers[production.type])(production) + end + local function visitn(productions) + local ret = {} + for keyword,production in pairs(productions) do + ret[keyword] = visit1(production) + end + return ret + end + function handlers.struct(production) + local member_names = {} + for k,_ in pairs(production.members) do table.insert(member_names, k) end + table.sort(member_names) + local emit_member = visitn(production.members) + return function(data, stream) + stream:write_stringref('tagged-struct') + stream:write_uint32(table_size(data)) + for _,k in ipairs(member_names) do + if data[k] ~= nil then + stream:write_stringref(k) + emit_member[k](data[k], stream) + end + end + end + end + function handlers.array(production) + local emit_tagged_value = visit1( + {type='scalar', argument_type=production.element_type}) + return function(data, stream) + stream:write_stringref('tagged-array') + stream:write_uint32(#data) + for i=1,#data do emit_tagged_value(data[i], stream) end + end + end + function handlers.table(production) + local emit_key = visit1({type='struct', members=production.keys}) + local emit_value = visit1({type='struct', members=production.values}) + return function(data, stream) + stream:write_stringref('tagged-table') + stream:write_uint32(#data) + -- FIXME: relies on data being an assoc + for _,v in ipairs(data) do + emit_key(v.key, stream) + emit_value(v.key, stream) + end + end + end + function handlers.scalar(production) + local primitive_type = production.argument_type.primitive_type + -- FIXME: needs a case for unions + if primitive_type == 'string' then + return function(data, stream) + stream:write_stringref('tagged-string') + stream:write_stringref(data) + end + else + local emit_value = value_emitter(primitive_type) + return function(data, stream) + stream:write_stringref('cdata') + stream:write_stringref(primitive_type) + emit_value(data, stream) + end + end + end + + return visit1(production) +end + +function data_compiler_from_grammar(emit_data, schema_name, schema_revision) + return function(data, filename, source_mtime) + source_mtime = source_mtime or {sec=0, nsec=0} + local stream = stream.open_temporary_output_byte_stream(filename) + local strtab = string_table_builder() + local header = header_t( + MAGIC, VERSION, source_mtime.sec, source_mtime.nsec, + strtab:intern(schema_name), strtab:intern(schema_revision or '')) + stream:write_ptr(header) -- Write with empty data_len etc, fix it later. + stream:align(4) + header.data_start = stream.written + local u32buf = ffi.new('uint32_t[1]') + function stream:write_uint32(val) + u32buf[0] = val + return self:write_ptr(u32buf) + end + function stream:write_stringref(str) + return self:write_uint32(strtab:intern(str)) + end + emit_data(data, stream) + header.data_len = stream.written - header.data_start + header.strtab_start, header.strtab_len = strtab:emit(stream) + stream:rewind() + -- Fix up header. + stream:write_ptr(header) + stream:close_and_rename() + end +end + +function data_compiler_from_schema(schema) + local grammar = data.data_grammar_from_schema(schema) + return data_compiler_from_grammar(data_emitter(grammar), + schema.id, schema.revision_date) +end + +function compile_data_for_schema(schema, data, filename, source_mtime) + return data_compiler_from_schema(schema)(data, filename, source_mtime) +end + +function compile_data_for_schema_by_name(schema_name, data, filename, source_mtime) + return compile_data_for_schema(schema.load_schema_by_name(schema_name), + data, filename, source_mtime) +end + +function selftest() + local test_schema = schema.load_schema([[module snabb-simple-router { + namespace snabb:simple-router; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + leaf active { type boolean; default true; } + + container routes { + presence true; + list route { + key addr; + leaf addr { type inet:ipv4-address; mandatory true; } + leaf port { type uint8 { range 0..11; } mandatory true; } + } + } + }]]) + local data = data.load_data_for_schema(test_schema, [[ + active true; + routes { + route { addr 1.2.3.4; port 1; } + route { addr 2.3.4.5; port 10; } + route { addr 3.4.5.6; port 2; } + } + ]]) + + local tmp = os.tmpname() + compile_data_for_schema(test_schema, data, tmp) + os.remove(tmp) +end diff --git a/src/lib/yang/stream.lua b/src/lib/yang/stream.lua new file mode 100644 index 0000000000..6446e0b70e --- /dev/null +++ b/src/lib/yang/stream.lua @@ -0,0 +1,167 @@ +module(..., package.seeall) + +local ffi = require("ffi") +local S = require("syscall") +local lib = require("core.lib") + +local function round_up(x, y) return y*math.ceil(x/y) end + +function open_output_byte_stream(filename) + local fd, err = + S.open(filename, "creat, wronly, trunc", "rusr, wusr, rgrp, roth") + if not fd then + error("error opening output file "..filename..": "..tostring(err)) + end + local ret = { written = 0, name = filename } + function ret:close() + fd:close() + end + function ret:error(msg) + self:close() + error('while writing file '..filename..': '..msg) + end + function ret:write(ptr, size) + assert(size) + ptr = ffi.cast("uint8_t*", ptr) + local to_write = size + while to_write > 0 do + local written, err = S.write(fd, ptr, to_write) + if not written then self:error(err) end + ptr = ptr + written + self.written = self.written + written + to_write = to_write - written + end + end + function ret:write_ptr(ptr) + self:align(ffi.alignof(ptr)) + self:write(ptr, ffi.sizeof(ptr)) + end + function ret:rewind() + fd:seek(0, 'set') + ret.written = 0 -- more of a position at this point + end + function ret:write_array(ptr, type, count) + self:align(ffi.alignof(type)) + self:write(ptr, ffi.sizeof(type) * count) + end + function ret:align(alignment) + local padding = round_up(self.written, alignment) - self.written + self:write(string.rep(' ', padding), padding) + end + return ret +end + +local function mktemp(name, mode) + if not mode then mode = "rusr, wusr, rgrp, roth" end + -- FIXME: If nothing seeds math.random, this produces completely + -- predictable numbers. + local t = math.random(1e7) + local tmpnam, fd, err + for i = t, t+10 do + tmpnam = name .. '.' .. i + fd, err = S.open(tmpnam, "creat, wronly, excl", mode) + if fd then + fd:close() + return tmpnam, nil + end + i = i + 1 + end + return nil, err +end + +function open_temporary_output_byte_stream(target) + local tmp_file, err = mktemp(target) + if not tmp_file then + local dir = lib.dirname(target) + error("failed to create temporary file in "..dir..": "..tostring(err)) + end + local stream = open_output_byte_stream(tmp_file) + function stream:close_and_rename() + self:close() + local res, err = S.rename(tmp_file, target) + if not res then + error("failed to rename "..tmp_file.." to "..target..": "..err) + end + end + return stream +end + +-- FIXME: Try to copy file into huge pages? +function open_input_byte_stream(filename) + local fd, err = S.open(filename, "rdonly") + if not fd then return + error("error opening "..filename..": "..tostring(err)) + end + local stat = S.fstat(fd) + local size = stat.size + local mem, err = S.mmap(nil, size, 'read, write', 'private', fd, 0) + fd:close() + if not mem then error("mmap failed: " .. tostring(err)) end + mem = ffi.cast("uint8_t*", mem) + local pos = 0 + local ret = { + name=filename, + mtime_sec=stat.st_mtime, + mtime_nsec=stat.st_mtime_nsec + } + function ret:close() + -- FIXME: Currently we don't unmap any memory. + -- S.munmap(mem, size) + mem, pos = nil, nil + end + function ret:error(msg) + error('while reading file '..filename..': '..msg) + end + function ret:read(count) + assert(count >= 0) + local ptr = mem + pos + pos = pos + count + if pos > size then + self:error('unexpected EOF') + end + return ptr + end + function ret:align(alignment) + self:read(round_up(pos, alignment) - pos) + end + function ret:seek(new_pos) + assert(new_pos >= 0) + assert(new_pos <= size) + pos = new_pos + end + function ret:read_ptr(type) + ret:align(ffi.alignof(type)) + return ffi.cast(ffi.typeof('$*', type), ret:read(ffi.sizeof(type))) + end + function ret:read_array(type, count) + ret:align(ffi.alignof(type)) + return ffi.cast(ffi.typeof('$*', type), + ret:read(ffi.sizeof(type) * count)) + end + function ret:read_char() + return ffi.string(ret:read(1), 1) + end + function ret:as_text_stream(len) + local end_pos = size + if len then end_pos = pos + len end + return { + name = ret.name, + mtime_sec = ret.mtime_sec, + mtime_nsec = ret.mtime_nsec, + read = function(self, n) + assert(n==1) + if pos == end_pos then return nil end + return ret:read_char() + end, + close = function() ret:close() end + } + end + return ret +end + +-- You're often better off using Lua's built-in files. This is here +-- because it gives a file-like object whose FD you can query, for +-- example to get its mtime. +function open_input_text_stream(filename) + return open_input_byte_stream(filename):as_text_stream() +end diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua index cf7787aa5a..16d39ee2a1 100644 --- a/src/lib/yang/value.lua +++ b/src/lib/yang/value.lua @@ -62,7 +62,7 @@ end types.binary = unimplemented('binary') types.bits = unimplemented('bits') -types.boolean = {} +types.boolean = {ctype='bool'} function types.boolean.parse(str, what) local str = assert(str, 'missing value for '..what) if str == 'true' then return true end From 4c86315c99ceffb538214acd2836689f80981bb8 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 2 Nov 2016 01:34:02 +0100 Subject: [PATCH 089/631] Refactor top.select_snabb_instance and fix small issues in claim_name This fixes a number of issues, some of which are cosmetic. The change refactors some of the "top" program's code for PID locating and selection. This change also cleans up a number of checks in claim_name into asserts. The big change is removal of the top level import of "core.app" into main. There doesn't seem to be usage of this paticular import in main and it causes stale data in "/var/run/snabb" as it creates "engine" shm directories for the supervisor. --- src/core/app.lua | 68 +++++++++-------------------------------- src/core/main.lua | 3 +- src/program/top/top.lua | 32 ++++++++++--------- 3 files changed, 34 insertions(+), 69 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index e16fc4f515..a7585ce0a0 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -128,84 +128,46 @@ function configure (new_config) counter.add(configs) end --- Claims a name for a program so it can be identified by name by other processes. +-- Claims a name for a program so it's identified by name by other processes. -- --- The name given to the function must be unique, if a name has been used before by --- an active process the function will error displaying an appropriate error message. --- Any other process can enumerate programs by their name and PID using enumerate_named_programs. +-- The name given to the function must be unique; if a name has been used before +-- by an active process the function will error displaying an appropriate error +-- message. The program can only claim one name, successive calls will produce +-- an error. function claim_name(name) - function name_from_path(dir) - -- Extracts last file/folder name from path - -- e.g. "/var/run/snabb/hello" -> "hello" - local ds, de = string.find(dir, "/[^/]*$") - return string.sub(dir, ds+1, de) - end - configuration[name] = name local namedir = "by-name/" .. name - local namedirqualified = named_program_root .. "/" .. name + local namedir_fq = named_program_root .. "/" .. name local piddir = shm.root .. "/" .. S.getpid() - local backlinkdir = piddir.."/name-backref" + local backlinkdir = piddir.."/name" -- Verify that the by-name directory exists. shm.mkdir(namedir) -- Verify that we've not already claimed a name - if S.lstat(backlinkdir) then - -- Read name of program for error message. - local fnamedir = S.readlink(backlinkdir) - local fname = name_from_path(fnamedir) - error("Program already claimed name '"..fname.."', cannot claim name '"..name.."'") - end - - -- Check if a symlink to a named program with the same name exists, if not, unlink it. - if S.lstat(namedirqualified) then - -- It exists, lets check what the PID is. - local piddir = S.readlink(namedirqualified) - local pid = tonumber(name_from_path(piddir)) - if S.kill(pid, 0) then - -- It's a running process, we should display an error message. - error("Name has been claimed by a currently running process: "..namedirqualified) - else - -- It's dead, we should be able to remove the symlink and relink. - S.unlink(namedirqualified) - end - end + assert(configuration.name, "Name already claimed, cannot claim: "..name) -- Create the new symlink. - local rtn, err = S.symlink(piddir, namedirqualified) - if rtn == -1 then - error("Error creating program name symlink in: "..namedirqualified) - end + assert(S.symlink(piddir, namedir_fq)) -- Create a backlink so to the symlink so we can easily cleanup - local rtn, err = S.symlink(namedirqualified, backlinkdir) - if rtn == -1 then - error("Error creating backlink for: "..namedirqualified) - end + assert(S.symlink(namedir_fq, backlinkdir)) end --- Enumerates all the named programs with their PID +-- Enumerates the named programs with their PID -- -- This returns a table programs with the key being the name of the program -- and the value being the PID of the program. Each program is checked that -- it's still alive. Any dead program or program without a name is not listed. --- --- The inc_dead argument is to disable the alive check (default: false) -function enumerate_named_programs(inc_dead) +function enumerate_named_programs() local progs = {} local dirs = shm.children("/by-name") if dirs == nil then return progs end for _, program in pairs(dirs) do local fq = named_program_root .. "/" .. program local piddir = S.readlink(fq) - local s, e = string.find(piddir, "/[^/]*$") - local pid = tonumber(string.sub(piddir, s+1, e)) - if inc_dead == true or S.kill(pid, 0) then - local ps, pe = string.find(fq, "/[^/]*$") - local program_name = string.sub(fq, ps+1, pe) - progs[program_name] = pid - end + local pid = tonumber(lib.basename(piddir)) + if S.kill(pid, 0) then progs[lib.basename(fq)] = pid end end return progs end @@ -620,7 +582,7 @@ function selftest () -- Ensure to claim two names fails assert(not pcall(claim_name, basename.."2")) - -- Lets check if they can be enumerated. + -- Check if it can be enumerated. local progs = enumerate_named_programs() assert(progs) assert(progs["testapp1"]) diff --git a/src/core/main.lua b/src/core/main.lua index 7b20a69019..f3bf425551 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -11,7 +11,6 @@ local ffi = require("ffi") local zone = require("jit.zone") local lib = require("core.lib") local shm = require("core.shm") -local app = require("core.app") local C = ffi.C -- Load ljsyscall early to help detect conflicts -- (e.g. FFI type name conflict between Snabb and ljsyscall) @@ -144,7 +143,7 @@ end function shutdown (pid) if not _G.developer_debug and not lib.getenv("SNABB_SHM_KEEP") then -- Check if there is a backlink to the named app, if so cleanup that. - local backlink = shm.root.."/"..pid.."/name-backref" + local backlink = shm.root.."/"..pid.."/name" if S.lstat(backlink) then local name_link = S.readlink(backlink) S.unlink(name_link) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 768154daa6..c464bb32d2 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -32,27 +32,31 @@ function run (args) end function select_snabb_instance (pid) - local instances = {} - -- For named programs there is a "by-name" directory which isn't a process, remove it. - for k,v in pairs(shm.children("/")) do if v ~= "by-name" then instances[k] = v end end + local function compute_snabb_instances() + -- Produces set of snabb instances, excluding this one. + local pids = {} + local my_pid = S.getpid() + for _, name in ipairs(shm.children("/")) do + -- This could fail as the name could be for example "by-name" + local p = tonumber(name) + if p and p ~= my_pid then table.insert(pids, p) end + end + return pids + end + + local instances = compute_snabb_instances() + if pid then -- Try to use given pid for _, instance in ipairs(instances) do if instance == pid then return pid end end print("No such Snabb instance: "..pid) - elseif #instances == 1 then print("No Snabb instance found.") + elseif #instances == 1 then return instances[1] + elseif #instances <= 0 then print("No Snabb instance found.") else - local own_pid = tostring(S.getpid()) - if #instances == 2 then - -- Two means one is us, so we pick the other. - return instances[1] == own_pid and instances[2] or instances[1] - else - print("Multiple Snabb instances found. Select one:") - for _, instance in ipairs(instances) do - if instance ~= own_pid then print(instance) end - end - end + print("Multiple Snabb instances found. Select one:") + for _, instance in ipairs(instances) do print(instance) end end main.exit(1) end From 68f6657824f5b0b65674d99b19417f8fa3d2fb55 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 10:31:59 +0100 Subject: [PATCH 090/631] Implement compiled configuration loader --- src/lib/yang/binary.lua | 102 +++++++++++++++++++++++++++++++++++----- src/lib/yang/data.lua | 2 +- src/lib/yang/stream.lua | 1 + 3 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index a5b30a4458..81d30c75f1 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -61,7 +61,6 @@ local function string_table_builder() end local function read_string_table(stream, strtab_len) - stream:align(4) assert(strtab_len >= 4) local count = stream:read_uint32() assert(strtab_len >= (4 * (count + 1))) @@ -72,7 +71,7 @@ local function read_string_table(stream, strtab_len) for i=0,count-1 do local len = offsets[i] - offset assert(len >= 0) - strings[i] = stream:read_string(len) + strings[i] = ffi.string(stream:read(len), len) offset = offset + len end return strings @@ -126,9 +125,7 @@ end -- -- code that uses the compiled table. local value_emitters = {} -local function value_emitter(primitive_type) - print(primitive_type) - local ctype = assert(assert(value.types[primitive_type]).ctype) +local function value_emitter(ctype) if value_emitters[ctype] then return value_emitters[ctype] end local type = ffi.typeof(ctype) local align = ffi.alignof(type) @@ -142,13 +139,13 @@ local function value_emitter(primitive_type) return emit end -function table_size(tab) +local function table_size(tab) local size = 0 for k,v in pairs(tab) do size = size + 1 end return size end -function data_emitter(production) +local function data_emitter(production) local handlers = {} local function visit1(production) return assert(handlers[production.type])(production) @@ -194,7 +191,7 @@ function data_emitter(production) -- FIXME: relies on data being an assoc for _,v in ipairs(data) do emit_key(v.key, stream) - emit_value(v.key, stream) + emit_value(v.value, stream) end end end @@ -207,10 +204,11 @@ function data_emitter(production) stream:write_stringref(data) end else - local emit_value = value_emitter(primitive_type) + local ctype = assert(assert(value.types[primitive_type]).ctype) + local emit_value = value_emitter(ctype) return function(data, stream) stream:write_stringref('cdata') - stream:write_stringref(primitive_type) + stream:write_stringref(ctype) emit_value(data, stream) end end @@ -228,7 +226,6 @@ function data_compiler_from_grammar(emit_data, schema_name, schema_revision) MAGIC, VERSION, source_mtime.sec, source_mtime.nsec, strtab:intern(schema_name), strtab:intern(schema_revision or '')) stream:write_ptr(header) -- Write with empty data_len etc, fix it later. - stream:align(4) header.data_start = stream.written local u32buf = ffi.new('uint32_t[1]') function stream:write_uint32(val) @@ -263,6 +260,78 @@ function compile_data_for_schema_by_name(schema_name, data, filename, source_mti data, filename, source_mtime) end +local function read_compiled_data(stream, strtab) + local function read_string() + return assert(strtab[stream:read_uint32()]) + end + local readers = {} + local function read1() + local tag = read_string() + return assert(readers[tag], tag)() + end + readers['tagged-struct'] = function () + local ret = {} + for i=1,stream:read_uint32() do + local k = read_string() + ret[k] = read1() + end + return ret + end + readers['tagged-array'] = function () + local ret = {} + for i=1,stream:read_uint32() do table.insert(ret, read1()) end + return ret + end + readers['tagged-table'] = function () + local ret = data.make_assoc() + for i=1,stream:read_uint32() do + local k = read1() + ret:add(k, read1()) + end + return ret + end + readers['tagged-string'] = function () + return read_string() + end + local ctypes = {} + local function scalar_type(ctype) + if not ctypes[ctype] then ctypes[ctype] = ffi.typeof(ctype) end + return ctypes[ctype] + end + readers['cdata'] = function () + local ctype = scalar_type(read_string()) + return stream:read_ptr(ctype)[0] + end + return read1() +end + +function load_compiled_data(stream) + local uint32_t = ffi.typeof('uint32_t') + function stream:read_uint32() + return stream:read_ptr(uint32_t)[0] + end + local header = stream:read_ptr(header_t) + assert(ffi.string(header.magic, ffi.sizeof(header.magic)) == MAGIC, + "expected file to begin with "..MAGIC) + assert(header.version == VERSION, + "incompatible version: "..header.version) + stream:seek(header.strtab_start) + local strtab = read_string_table(stream, header.strtab_len) + local ret = {} + ret.schema_name = strtab[header.schema_name] + ret.revision_date = strtab[header.revision_date] + ret.source_mtime = {sec=header.source_mtime_sec, + nsec=header.source_mtime_nsec} + stream:seek(header.data_start) + ret.data = read_compiled_data(stream, strtab) + assert(stream:seek() == header.data_start + header.data_len) + return ret +end + +function load_compiled_data_file(filename) + return load_compiled_data(stream.open_input_byte_stream(filename)) +end + function selftest() local test_schema = schema.load_schema([[module snabb-simple-router { namespace snabb:simple-router; @@ -292,5 +361,16 @@ function selftest() local tmp = os.tmpname() compile_data_for_schema(test_schema, data, tmp) + local data2 = load_compiled_data_file(tmp) os.remove(tmp) + + local ipv4 = require('lib.protocol.ipv4') + assert(data2.schema_name == 'snabb-simple-router') + assert(data2.revision_date == '') + assert(data2.data.active == true) + -- These tests don't work yet because we need to fix our + -- associative data structure to usefully allow cdata keys. + -- assert(data2.data.routes.route:get_value(ipv4:pton('1.2.3.4')).port == 1) + -- assert(data2.data.routes.route:get_value(ipv4:pton('2.3.4.5')).port == 10) + -- assert(data2.data.routes.route:get_value(ipv4:pton('3.4.5.6')).port == 2) end diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 8f96ab8d65..0fbda9fd66 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -164,7 +164,7 @@ end -- Simple temporary associative array until we get the various Table -- kinds working. -local function make_assoc() +function make_assoc() local assoc = {} function assoc:get_entry(k) assert(type(k) ~= 'table', 'multi-key lookup unimplemented') diff --git a/src/lib/yang/stream.lua b/src/lib/yang/stream.lua index 6446e0b70e..14f3925ac7 100644 --- a/src/lib/yang/stream.lua +++ b/src/lib/yang/stream.lua @@ -125,6 +125,7 @@ function open_input_byte_stream(filename) self:read(round_up(pos, alignment) - pos) end function ret:seek(new_pos) + if new_pos == nil then return pos end assert(new_pos >= 0) assert(new_pos <= size) pos = new_pos From 9b9eb69c8aa5da6cc57bfaaf5dda70dc95041b5c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 11:11:22 +0100 Subject: [PATCH 091/631] Add choose-representations pass This is in preparation for representing YANG list data as ctables, if possible. --- src/lib/yang/binary.lua | 12 ++++----- src/lib/yang/data.lua | 54 +++++++++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 81d30c75f1..766212385f 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -157,7 +157,7 @@ local function data_emitter(production) end return ret end - function handlers.struct(production) + function handlers.tagged_struct(production) local member_names = {} for k,_ in pairs(production.members) do table.insert(member_names, k) end table.sort(member_names) @@ -173,7 +173,7 @@ local function data_emitter(production) end end end - function handlers.array(production) + function handlers.tagged_array(production) local emit_tagged_value = visit1( {type='scalar', argument_type=production.element_type}) return function(data, stream) @@ -182,9 +182,9 @@ local function data_emitter(production) for i=1,#data do emit_tagged_value(data[i], stream) end end end - function handlers.table(production) - local emit_key = visit1({type='struct', members=production.keys}) - local emit_value = visit1({type='struct', members=production.values}) + function handlers.tagged_tagged_table(production) + local emit_key = visit1({type='tagged_struct', members=production.keys}) + local emit_value = visit1({type='tagged_struct', members=production.values}) return function(data, stream) stream:write_stringref('tagged-table') stream:write_uint32(#data) @@ -214,7 +214,7 @@ local function data_emitter(production) end end - return visit1(production) + return visit1(data.choose_representations(production)) end function data_compiler_from_grammar(emit_data, schema_name, schema_revision) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 0fbda9fd66..c7dfcec9ab 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -100,7 +100,7 @@ local function assert_not_duplicate(out, keyword) assert(not out, 'duplicate parameter: '..keyword) end -local function struct_parser(keyword, members) +local function tagged_struct_parser(keyword, members) local function init() return nil end local function parse1(node) assert_compound(node, keyword) @@ -125,7 +125,7 @@ local function struct_parser(keyword, members) return {init=init, parse=parse, finish=finish} end -local function array_parser(keyword, element_type) +local function tagged_array_parser(keyword, element_type) local function init() return {} end local parsev = value_parser(element_type) local function parse1(node) @@ -184,12 +184,11 @@ function make_assoc() return assoc end -local function table_parser(keyword, keys, values) +local function tagged_tagged_table_parser(keyword, keys, values) local members = {} for k,v in pairs(keys) do members[k] = v end for k,v in pairs(values) do members[k] = v end - local parser = struct_parser(keyword, members) - + local parser = tagged_struct_parser(keyword, members) local function init() return make_assoc() end local function parse1(node) assert_compound(node, keyword) @@ -214,6 +213,35 @@ function data_parser_from_schema(schema) return data_parser_from_grammar(data_grammar_from_schema(schema)) end +function choose_representations(production) + local handlers = {} + local function visit1(production) + return assert(handlers[production.type])(production) + end + local function visitn(productions) + local ret = {} + for keyword,production in pairs(productions) do + ret[keyword] = visit1(production) + end + return ret + end + function handlers.struct(production) + return {type='tagged_struct', members=visitn(production.members)} + end + function handlers.array(production) + return {type='tagged_array', element_type=production.element_type} + end + function handlers.table(production) + return {type='tagged_tagged_table',keys=visitn(production.keys), + values = visitn(production.values)} + end + function handlers.scalar(production) + return {type='scalar', argument_type=production.argument_type, + default=production.default, mandatory=production.mandatory} + end + return visit1(production) +end + function data_parser_from_grammar(production) local handlers = {} local function visit1(keyword, production) @@ -226,22 +254,22 @@ function data_parser_from_grammar(production) end return ret end - function handlers.struct(keyword, production) - return struct_parser(keyword, visitn(production.members)) + function handlers.tagged_struct(keyword, production) + return tagged_struct_parser(keyword, visitn(production.members)) end - function handlers.array(keyword, production) - return array_parser(keyword, production.element_type) + function handlers.tagged_array(keyword, production) + return tagged_array_parser(keyword, production.element_type) end - function handlers.table(keyword, production) - return table_parser(keyword, visitn(production.keys), - visitn(production.values)) + function handlers.tagged_tagged_table(keyword, production) + return tagged_tagged_table_parser(keyword, visitn(production.keys), + visitn(production.values)) end function handlers.scalar(keyword, production) return scalar_parser(keyword, production.argument_type, production.default, production.mandatory) end - local parser = visit1('(top level)', production) + local parser = visit1('(top level)', choose_representations(production)) return function(str, filename) local node = {statements=parse_string(str, filename)} return parser.finish(parser.parse(node, parser.init())) From b863974e0b8c0823225dfa458afab07b07f25d47 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 12:15:21 +0100 Subject: [PATCH 092/631] Rework representation-choosing Instead of specializing the grammar, we just add "ctype" entries if we think that a struct, array, or table should be raw. --- src/lib/yang/binary.lua | 70 ++++++++++++++++---------- src/lib/yang/data.lua | 106 ++++++++++++++++++---------------------- 2 files changed, 91 insertions(+), 85 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 766212385f..99e0c27ee8 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -157,41 +157,57 @@ local function data_emitter(production) end return ret end - function handlers.tagged_struct(production) + function handlers.struct(production) local member_names = {} for k,_ in pairs(production.members) do table.insert(member_names, k) end table.sort(member_names) - local emit_member = visitn(production.members) - return function(data, stream) - stream:write_stringref('tagged-struct') - stream:write_uint32(table_size(data)) - for _,k in ipairs(member_names) do - if data[k] ~= nil then - stream:write_stringref(k) - emit_member[k](data[k], stream) + if production.ctype then + error('unimplemented') + else + local emit_member = visitn(production.members) + return function(data, stream) + stream:write_stringref('tagged-struct') + stream:write_uint32(table_size(data)) + for _,k in ipairs(member_names) do + if data[k] ~= nil then + stream:write_stringref(k) + emit_member[k](data[k], stream) + end end end end end - function handlers.tagged_array(production) - local emit_tagged_value = visit1( - {type='scalar', argument_type=production.element_type}) - return function(data, stream) - stream:write_stringref('tagged-array') - stream:write_uint32(#data) - for i=1,#data do emit_tagged_value(data[i], stream) end + function handlers.array(production) + if production.ctype then + error('unimplemented') + else + local emit_tagged_value = visit1( + {type='scalar', argument_type=production.element_type}) + return function(data, stream) + stream:write_stringref('tagged-array') + stream:write_uint32(#data) + for i=1,#data do emit_tagged_value(data[i], stream) end + end end end - function handlers.tagged_tagged_table(production) - local emit_key = visit1({type='tagged_struct', members=production.keys}) - local emit_value = visit1({type='tagged_struct', members=production.values}) - return function(data, stream) - stream:write_stringref('tagged-table') - stream:write_uint32(#data) - -- FIXME: relies on data being an assoc - for _,v in ipairs(data) do - emit_key(v.key, stream) - emit_value(v.value, stream) + function handlers.table(production) + if production.key_ctype and production.value_ctype then + error('unimplemented') + elseif production.key_ctype then + error('unimplemented') + elseif production.value_ctype then + error('unimplemented') + else + local emit_key = visit1({type='struct', members=production.keys}) + local emit_value = visit1({type='struct', members=production.values}) + return function(data, stream) + stream:write_stringref('tagged-table') + stream:write_uint32(#data) + -- FIXME: relies on data being an assoc + for _,v in ipairs(data) do + emit_key(v.key, stream) + emit_value(v.value, stream) + end end end end @@ -214,7 +230,7 @@ local function data_emitter(production) end end - return visit1(data.choose_representations(production)) + return visit1(production) end function data_compiler_from_grammar(emit_data, schema_name, schema_revision) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index c7dfcec9ab..3b72c850a4 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -7,14 +7,9 @@ local schema = require("lib.yang.schema") local util = require("lib.yang.util") local value = require("lib.yang.value") --- FIXME: --- Parse inet:mac-address using ethernet:pton --- Parse inet:ipv4-address using ipv4:pton --- Parse inet:ipv6-address using ipv6:pton --- Parse inet:ipv4-prefix? --- Parse inet:ipv6-prefix? - function data_grammar_from_schema(schema) + local function struct_ctype(members) end + local function value_ctype(type) end local handlers = {} local function visit(node) local handler = handlers[node.kind] @@ -32,11 +27,14 @@ function data_grammar_from_schema(schema) return ret end function handlers.container(node) - if not node.presence then return visit_body(node) end - return {[node.id]={type='struct', members=visit_body(node)}} + local members = visit_body(node) + if not node.presence then return members end + return {[node.id]={type='struct', members=members, + ctype=struct_ctype(members)}} end handlers['leaf-list'] = function(node) - return {[node.id]={type='array', element_type=node.type}} + return {[node.id]={type='array', element_type=node.type, + ctype=value_ctype(node.type)}} end function handlers.list(node) local members=visit_body(node) @@ -45,13 +43,17 @@ function data_grammar_from_schema(schema) for k,v in pairs(members) do if not keys[k] then values[k] = v end end - return {[node.id]={type='table', keys=keys, values=values}} + return {[node.id]={type='table', keys=keys, values=values, + key_ctype=struct_ctype(keys), + value_ctype=struct_ctype(values)}} end function handlers.leaf(node) return {[node.id]={type='scalar', argument_type=node.type, - default=node.default, mandatory=node.mandatory}} + default=node.default, mandatory=node.mandatory, + ctype=value_ctype(node.type)}} end - return {type="struct", members=visit_body(schema)} + local members = visit_body(schema) + return {type="struct", members=members, ctype=struct_ctype(members)} end local function integer_type(min, max) @@ -100,7 +102,7 @@ local function assert_not_duplicate(out, keyword) assert(not out, 'duplicate parameter: '..keyword) end -local function tagged_struct_parser(keyword, members) +local function struct_parser(keyword, members, ctype) local function init() return nil end local function parse1(node) assert_compound(node, keyword) @@ -118,14 +120,15 @@ local function tagged_struct_parser(keyword, members) assert_not_duplicate(out, keyword) return parse1(node) end + local struct_t = ctype and ffi.typeof(ctype) local function finish(out) - -- FIXME check mandatory - return out + -- FIXME check mandatory values. + if struct_t then return struct_t(out) else return out end end return {init=init, parse=parse, finish=finish} end -local function tagged_array_parser(keyword, element_type) +local function array_parser(keyword, element_type, ctype) local function init() return {} end local parsev = value_parser(element_type) local function parse1(node) @@ -136,9 +139,10 @@ local function tagged_array_parser(keyword, element_type) table.insert(out, parse1(node)) return out end + local array_t = ctype and ffi.typeof('$[?]', ffi.typeof(ctype)) local function finish(out) -- FIXME check min-elements - return out + if array_t then return array_t(out) else return out end end return {init=init, parse=parse, finish=finish} end @@ -184,12 +188,23 @@ function make_assoc() return assoc end -local function tagged_tagged_table_parser(keyword, keys, values) +local function table_parser(keyword, keys, values, key_ctype, value_ctype) local members = {} for k,v in pairs(keys) do members[k] = v end for k,v in pairs(values) do members[k] = v end - local parser = tagged_struct_parser(keyword, members) - local function init() return make_assoc() end + local parser = struct_parser(keyword, members) + local key_t = key_ctype and ffi.typeof(key_ctype) + local value_t = value_ctype and ffi.typeof(value_ctype) + local init + if key_t and value_t then + function init() + return ctable.new({ key_type=key_t, value_type=value_t }) + end + elseif key_t or value_t then + error('mixed table types currently unimplemented') + else + function init() return make_assoc() end + end local function parse1(node) assert_compound(node, keyword) return parser.finish(parser.parse(node, parser.init())) @@ -197,6 +212,8 @@ local function tagged_tagged_table_parser(keyword, keys, values) local function parse(node, assoc) local struct = parse1(node) local key, value = {}, {} + if key_t then key = key_t() end + if value_t then value = value_t() end for k,v in pairs(struct) do if keys[k] then key[k] = v else value[k] = v end end @@ -213,35 +230,6 @@ function data_parser_from_schema(schema) return data_parser_from_grammar(data_grammar_from_schema(schema)) end -function choose_representations(production) - local handlers = {} - local function visit1(production) - return assert(handlers[production.type])(production) - end - local function visitn(productions) - local ret = {} - for keyword,production in pairs(productions) do - ret[keyword] = visit1(production) - end - return ret - end - function handlers.struct(production) - return {type='tagged_struct', members=visitn(production.members)} - end - function handlers.array(production) - return {type='tagged_array', element_type=production.element_type} - end - function handlers.table(production) - return {type='tagged_tagged_table',keys=visitn(production.keys), - values = visitn(production.values)} - end - function handlers.scalar(production) - return {type='scalar', argument_type=production.argument_type, - default=production.default, mandatory=production.mandatory} - end - return visit1(production) -end - function data_parser_from_grammar(production) local handlers = {} local function visit1(keyword, production) @@ -254,22 +242,24 @@ function data_parser_from_grammar(production) end return ret end - function handlers.tagged_struct(keyword, production) - return tagged_struct_parser(keyword, visitn(production.members)) + function handlers.struct(keyword, production) + local members = visitn(production.members) + return struct_parser(keyword, members, production.ctype) end - function handlers.tagged_array(keyword, production) - return tagged_array_parser(keyword, production.element_type) + function handlers.array(keyword, production) + return array_parser(keyword, production.element_type, production.ctype) end - function handlers.tagged_tagged_table(keyword, production) - return tagged_tagged_table_parser(keyword, visitn(production.keys), - visitn(production.values)) + function handlers.table(keyword, production) + local keys, values = visitn(production.keys), visitn(production.values) + return table_parser(keyword, keys, values, production.key_ctype, + production.value_ctype) end function handlers.scalar(keyword, production) return scalar_parser(keyword, production.argument_type, production.default, production.mandatory) end - local parser = visit1('(top level)', choose_representations(production)) + local parser = visit1('(top level)', production) return function(str, filename) local node = {statements=parse_string(str, filename)} return parser.finish(parser.parse(node, parser.init())) From 7ffbce78a0cc53631eae6c5bd1444b5e0fda6dbf Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 12:57:40 +0100 Subject: [PATCH 093/631] Normalize configuration identifiers A container foo with a leaf "is-active" will now access the value as foo.is_active, not foo['is-active']. This also helps us make identifiers that can be embedded in C struct types. --- src/lib/yang/binary.lua | 14 ++++++++------ src/lib/yang/data.lua | 29 ++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 99e0c27ee8..c80341cd70 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -165,13 +165,15 @@ local function data_emitter(production) error('unimplemented') else local emit_member = visitn(production.members) + local normalize_id = data.normalize_id return function(data, stream) stream:write_stringref('tagged-struct') stream:write_uint32(table_size(data)) for _,k in ipairs(member_names) do - if data[k] ~= nil then - stream:write_stringref(k) - emit_member[k](data[k], stream) + local id = normalize_id(k) + if data[id] ~= nil then + stream:write_stringref(id) + emit_member[k](data[id], stream) end end end @@ -355,7 +357,7 @@ function selftest() import ietf-inet-types {prefix inet;} - leaf active { type boolean; default true; } + leaf is-active { type boolean; default true; } container routes { presence true; @@ -367,7 +369,7 @@ function selftest() } }]]) local data = data.load_data_for_schema(test_schema, [[ - active true; + is-active true; routes { route { addr 1.2.3.4; port 1; } route { addr 2.3.4.5; port 10; } @@ -383,7 +385,7 @@ function selftest() local ipv4 = require('lib.protocol.ipv4') assert(data2.schema_name == 'snabb-simple-router') assert(data2.revision_date == '') - assert(data2.data.active == true) + assert(data2.data.is_active == true) -- These tests don't work yet because we need to fix our -- associative data structure to usefully allow cdata keys. -- assert(data2.data.routes.route:get_value(ipv4:pton('1.2.3.4')).port == 1) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 3b72c850a4..42fc3d2758 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -7,6 +7,10 @@ local schema = require("lib.yang.schema") local util = require("lib.yang.util") local value = require("lib.yang.value") +function normalize_id(id) + return id:gsub('[^%w_]', '_') +end + function data_grammar_from_schema(schema) local function struct_ctype(members) end local function value_ctype(type) end @@ -21,6 +25,8 @@ function data_grammar_from_schema(schema) for id,node in pairs(node.body) do for keyword,node in pairs(visit(node)) do assert(not ret[keyword], 'duplicate identifier: '..keyword) + assert(not ret[normalize_id(keyword)], + 'duplicate identifier: '..normalize_id(keyword)) ret[keyword] = node end end @@ -107,13 +113,17 @@ local function struct_parser(keyword, members, ctype) local function parse1(node) assert_compound(node, keyword) local ret = {} - for k,sub in pairs(members) do ret[k] = sub.init() end + for k,sub in pairs(members) do ret[normalize_id(k)] = sub.init() end for _,node in ipairs(node.statements) do local sub = assert(members[node.keyword], 'unrecognized parameter: '..node.keyword) - ret[node.keyword] = sub.parse(node, ret[node.keyword]) + local id = normalize_id(node.keyword) + ret[id] = sub.parse(node, ret[id]) + end + for k,sub in pairs(members) do + local id = normalize_id(k) + ret[id] = sub.finish(ret[id]) end - for k,sub in pairs(members) do ret[k] = sub.finish(ret[k]) end return ret end local function parse(node, out) @@ -215,7 +225,8 @@ local function table_parser(keyword, keys, values, key_ctype, value_ctype) if key_t then key = key_t() end if value_t then value = value_t() end for k,v in pairs(struct) do - if keys[k] then key[k] = v else value[k] = v end + local id = normalize_id(k) + if keys[k] then key[id] = v else value[id] = v end end assoc:add(key, value) return assoc @@ -312,18 +323,18 @@ function selftest() } addr 1.2.3.4; ]]) - assert(data['fruit-bowl'].description == 'ohai') - local contents = data['fruit-bowl'].contents + assert(data.fruit_bowl.description == 'ohai') + local contents = data.fruit_bowl.contents assert(contents:get_entry('foo').key.name == 'foo') assert(contents:get_entry('foo').value.score == 7) assert(contents:get_key('foo').name == 'foo') assert(contents:get_value('foo').score == 7) - assert(contents:get_value('foo')['tree-grown'] == nil) + assert(contents:get_value('foo').tree_grown == nil) assert(contents:get_key('bar').name == 'bar') assert(contents:get_value('bar').score == 8) - assert(contents:get_value('bar')['tree-grown'] == nil) + assert(contents:get_value('bar').tree_grown == nil) assert(contents:get_key('baz').name == 'baz') assert(contents:get_value('baz').score == 9) - assert(contents:get_value('baz')['tree-grown'] == true) + assert(contents:get_value('baz').tree_grown == true) assert(require('lib.protocol.ipv4'):ntop(data.addr) == '1.2.3.4') end From 580d8c7ee7b118c1bda7da9b3d941f27fbf9ee2b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 13:43:51 +0100 Subject: [PATCH 094/631] ctable: hash_fn param is optional If no hash_fn is given, ctable will provide one. --- src/lib/README.ctable.md | 10 +++++++++- src/lib/ctable.lua | 25 +++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md index d0178212a5..355626fb8b 100644 --- a/src/lib/README.ctable.md +++ b/src/lib/README.ctable.md @@ -62,7 +62,6 @@ following keys are required: * `key_type`: An FFI type (LuaJIT "ctype") for keys in this table. * `value_type`: An FFI type (LuaJT "ctype") for values in this table. - * `hash_fn`: A function that takes a key and returns a hash value. Hash values are unsigned 32-bit integers in the range `[0, 0xFFFFFFFF)`. That is to say, `0xFFFFFFFF` is the only unsigned 32-bit @@ -71,6 +70,9 @@ hash value in the correct range. Optional entries that may be present in the *parameters* table include: + * `hash_fn`: A function that takes a key and returns a hash value. + If not given, defaults to the result of calling `compute_hash_fn` + on the key type. * `initial_size`: The initial size of the hash table, including free space. Defaults to 8 slots. * `max_occupancy_rate`: The maximum ratio of `occupancy/size`, where @@ -198,3 +200,9 @@ Hash the first 48 bits of a byte sequence. — Function **ctable.hashv_64** *ptr* Hash the first 64 bits of a byte sequence. + +— Function **ctable.compute_hash_fn** *ctype* + +Return a `hashv_`-like hash function over the bytes in instances of +*ctype*. Note that the same reservations apply as for `hash_32` +above. diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index b167b59570..4abc189955 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -91,8 +91,9 @@ end -- FIXME: For now the value_type option is required, but in the future -- we should allow for a nil value type to create a set instead of a -- map. -local required_params = set('key_type', 'value_type', 'hash_fn') +local required_params = set('key_type', 'value_type') local optional_params = { + hash_fn = false, initial_size = 8, max_occupancy_rate = 0.9, min_occupancy_rate = 0.0 @@ -103,7 +104,7 @@ function new(params) local params = parse_params(params, required_params, optional_params) ctab.entry_type = make_entry_type(params.key_type, params.value_type) ctab.type = make_entries_type(ctab.entry_type) - ctab.hash_fn = params.hash_fn + ctab.hash_fn = params.hash_fn or compute_hash_fn(params.key_type) ctab.equal_fn = make_equal_fn(params.key_type) ctab.size = 0 ctab.occupancy = 0 @@ -391,6 +392,7 @@ function hash_32(i32) return uint32_cast[0] end +local cast = ffi.cast function hashv_32(key) return hash_32(cast(uint32_ptr_t, key)[0]) end @@ -410,6 +412,22 @@ function hashv_64(key) return hash_32(bxor(hi, hash_32(lo))) end +local hash_fns_by_size = { [4]=hashv_32, [8]=hashv_64 } +function compute_hash_fn(ctype) + local size = ffi.sizeof(ctype) + if not hash_fns_by_size[size] then + hash_fns_by_size[size] = function(key) + local h = 0 + local words = cast(uint32_ptr_t, key) + local bytes = cast('uint8_t*', key) + for i=0,size/4 do h = hash_32(bxor(h, words[i])) end + for i=1,size%4 do h = hash_32(bxor(h, bytes[size-i])) end + return h + end + end + return hash_fns_by_size[size] +end + function selftest() print("selftest: ctable") @@ -457,8 +475,11 @@ function selftest() local function check_bytes_equal(type, a, b) local equal_fn = make_equal_fn(type) + local hash_fn = compute_hash_fn(type) assert(equal_fn(ffi.new(type, a), ffi.new(type, a))) assert(not equal_fn(ffi.new(type, a), ffi.new(type, b))) + assert(hash_fn(ffi.new(type, a)) == hash_fn(ffi.new(type, a))) + assert(hash_fn(ffi.new(type, a)) ~= hash_fn(ffi.new(type, b))) end check_bytes_equal(ffi.typeof('uint16_t[1]'), {1}, {2}) -- 2 byte check_bytes_equal(ffi.typeof('uint32_t[1]'), {1}, {2}) -- 4 byte From 7962515cf2ce3d297e459bd4fc86d51026091558 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 2 Nov 2016 13:49:10 +0100 Subject: [PATCH 095/631] Fixes error in selftest for core.app The assert was checking the opposite of what it was suppose to, failing when things were correct. This fixes it to ensure it is checking that program has NOT got a name assiged. --- src/core/app.lua | 2 +- src/core/main.lua | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index a7585ce0a0..88e3b1b7cb 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -145,7 +145,7 @@ function claim_name(name) shm.mkdir(namedir) -- Verify that we've not already claimed a name - assert(configuration.name, "Name already claimed, cannot claim: "..name) + assert(configuration.name == nil, "Name already claimed, cannot claim: "..name) -- Create the new symlink. assert(S.symlink(piddir, namedir_fq)) diff --git a/src/core/main.lua b/src/core/main.lua index f3bf425551..bd8e3e6e9a 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -142,13 +142,11 @@ end -- Cleanup after Snabb process. function shutdown (pid) if not _G.developer_debug and not lib.getenv("SNABB_SHM_KEEP") then - -- Check if there is a backlink to the named app, if so cleanup that. + -- Try cleaning up symlinks for named apps, if none exist, fail silently. local backlink = shm.root.."/"..pid.."/name" - if S.lstat(backlink) then - local name_link = S.readlink(backlink) - S.unlink(name_link) - S.unlink(backlink) - end + local name_link = S.readlink(backlink) + S.unlink(name_link) + S.unlink(backlink) shm.unlink("/"..pid) end From 6177659db8e69f4772c78d5488c2685fb4121fda Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 14:43:59 +0100 Subject: [PATCH 096/631] Ctable supports load/save This allows a ctable to be persisted to disk or to a binary stream and cheaply loaded back again at run-time. --- src/lib/README.ctable.md | 16 ++++++ src/lib/ctable.lua | 102 ++++++++++++++++++++++++++++++++++----- 2 files changed, 106 insertions(+), 12 deletions(-) diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md index 355626fb8b..17c79b9f7a 100644 --- a/src/lib/README.ctable.md +++ b/src/lib/README.ctable.md @@ -83,6 +83,15 @@ Optional entries that may be present in the *parameters* table include: * `min_occupancy_rate`: Minimum ratio of `occupancy/size`. Removing an entry from an "empty" table will shrink the table. +— Function **ctable.load** *stream* *parameters* + +Load a ctable that was previously saved out to a binary format. +*parameters* are as for `ctable.new`. *stream* should be an object +that has a **:read_ptr**(*ctype*) method, which returns a pointer to +an embedded instances of *ctype* in the stream, advancing the stream +over the object; and **:read_array**(*ctype*, *count*) which is the +same but reading *count* instances of *ctype* instead of just one. + #### Methods Users interact with a ctable through methods. In these method @@ -156,6 +165,13 @@ Return true if we actually do find a value and remove it. Otherwise if no entry is found in the table and *missing_allowed* is true, then return false. Otherwise raise an error. +— Method **:save** *stream* + +Save a ctable to a byte sink. *stream* should be an object that has a +**:write_ptr**(*ctype*) method, which writes an instance of a struct +type out to a stream, and **:write_array**(*ctype*, *count*) which is +the same but writing *count* instances of *ctype* instead of just one. + — Method **:selfcheck** Run an expensive internal diagnostic to verify that the table's internal diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 4abc189955..11e6b9ba37 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -168,6 +168,45 @@ function CTable:get_backing_size() return self.byte_size end +local header_t = ffi.typeof[[ +struct { + uint32_t size; + uint32_t occupancy; + uint32_t max_displacement; + double max_occupancy_rate; + double min_occupancy_rate; +} +]] + +function load(stream, params) + local header = stream:read_ptr(header_t) + local params_copy = {} + for k,v in pairs(params) do params_copy[k] = v end + params_copy.initial_size = header.size + params_copy.min_occupancy_rate = header.min_occupancy_rate + params_copy.max_occupancy_rate = header.max_occupancy_rate + local ctab = new(params_copy) + ctab.occupancy = header.occupancy + ctab.max_displacement = header.max_displacement + local entry_count = ctab.size + ctab.max_displacement + 1 + + -- Slurp the entries directly into the ctable's backing store. + -- This ensures that the ctable is in hugepages. + C.memcpy(ctab.entries, + stream:read_array(ctab.entry_type, entry_count), + ffi.sizeof(ctab.entry_type) * entry_count) + + return ctab +end + +function CTable:save(stream) + stream:write_ptr(header_t(self.size, self.occupancy, self.max_displacement, + self.max_occupancy_rate, self.min_occupancy_rate)) + stream:write_array(self.entries, + self.entry_type, + self.size + self.max_displacement + 1) +end + function CTable:insert(hash, key, value, updates_allowed) if self.occupancy + 1 > self.occupancy_hi then self:resize(self.size * 2) @@ -450,21 +489,60 @@ function selftest() ctab:add(i, v) end - -- In this case we know max_displacement is 8. Assert here so that - -- we can detect any future deviation or regression. - assert(ctab.max_displacement == 8) + for i=1,2 do + -- In this case we know max_displacement is 8. Assert here so that + -- we can detect any future deviation or regression. + assert(ctab.max_displacement == 8) - ctab:selfcheck() + ctab:selfcheck() - for i = 1, occupancy do - local value = ctab:lookup_ptr(i).value[0] - assert(value == bnot(i)) - end - ctab:selfcheck() + for i = 1, occupancy do + local value = ctab:lookup_ptr(i).value[0] + assert(value == bnot(i)) + end + ctab:selfcheck() + + local iterated = 0 + for entry in ctab:iterate() do iterated = iterated + 1 end + assert(iterated == occupancy) - local iterated = 0 - for entry in ctab:iterate() do iterated = iterated + 1 end - assert(iterated == occupancy) + -- Save the table out to disk, reload it, and run the same + -- checks. + local tmp = os.tmpname() + do + local file = io.open(tmp, 'wb') + local function write(ptr, size) + + file:write(ffi.string(ptr, size)) + end + local stream = {} + function stream:write_ptr(ptr) + write(ptr, ffi.sizeof(ptr)) + end + function stream:write_array(ptr, type, count) + write(ptr, ffi.sizeof(type) * count) + end + ctab:save(stream) + file:close() + end + do + local file = io.open(tmp, 'rb') + local function read(size) + return ffi.new('uint8_t[?]', size, file:read(size)) + end + local stream = {} + function stream:read_ptr(type) + return ffi.cast(ffi.typeof('$*', type), read(ffi.sizeof(type))) + end + function stream:read_array(type, count) + return ffi.cast(ffi.typeof('$*', type), + read(ffi.sizeof(type) * count)) + end + ctab = load(stream, params) + file:close() + end + os.remove(tmp) + end -- OK, all looking good with our ctab. From 96b605c966c61951199612220b57cc0feb7bfb85 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 14:45:36 +0100 Subject: [PATCH 097/631] YANG configurations compiled as raw ctables For "list" types whose keys and values are of fixed size and amenable to being stored in ctables, that's what we do. --- src/lib/yang/binary.lua | 49 +++++++++++++++++++++++++++++------------ src/lib/yang/data.lua | 34 +++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index c80341cd70..d7b14a20eb 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -8,6 +8,7 @@ local util = require("lib.yang.util") local value = require("lib.yang.value") local stream = require("lib.yang.stream") local data = require('lib.yang.data') +local ctable = require('lib.ctable') local MAGIC = "yangconf" local VERSION = 0x00001000 @@ -194,12 +195,19 @@ local function data_emitter(production) end function handlers.table(production) if production.key_ctype and production.value_ctype then - error('unimplemented') - elseif production.key_ctype then - error('unimplemented') - elseif production.value_ctype then - error('unimplemented') + return function(data, stream) + stream:write_stringref('ctable') + stream:write_stringref(production.key_ctype) + stream:write_stringref(production.value_ctype) + data:save(stream) + end else + -- TODO: here we should implement mixed table types if key_t or + -- value_t is non-nil, or string-keyed tables if the key is a + -- string. For the moment, fall back to the old assoc + -- implementation. + -- if production.key_ctype then error('unimplemented') end + -- if production.value_ctype then error('unimplemented') end local emit_key = visit1({type='struct', members=production.keys}) local emit_value = visit1({type='struct', members=production.values}) return function(data, stream) @@ -308,6 +316,12 @@ local function read_compiled_data(stream, strtab) end return ret end + readers['ctable'] = function () + local key_ctype = read_string() + local value_ctype = read_string() + local key_t, value_t = ffi.typeof(key_ctype), ffi.typeof(value_ctype) + return ctable.load(stream, {key_type=key_t, value_type=value_t}) + end readers['tagged-string'] = function () return read_string() end @@ -377,18 +391,25 @@ function selftest() } ]]) + local ipv4 = require('lib.protocol.ipv4') + + assert(data.is_active == true) + local routing_table = data.routes.route + assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) + assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) + assert(routing_table:lookup_ptr(ipv4:pton('3.4.5.6')).value.port == 2) + local tmp = os.tmpname() compile_data_for_schema(test_schema, data, tmp) local data2 = load_compiled_data_file(tmp) - os.remove(tmp) - - local ipv4 = require('lib.protocol.ipv4') assert(data2.schema_name == 'snabb-simple-router') assert(data2.revision_date == '') - assert(data2.data.is_active == true) - -- These tests don't work yet because we need to fix our - -- associative data structure to usefully allow cdata keys. - -- assert(data2.data.routes.route:get_value(ipv4:pton('1.2.3.4')).port == 1) - -- assert(data2.data.routes.route:get_value(ipv4:pton('2.3.4.5')).port == 10) - -- assert(data2.data.routes.route:get_value(ipv4:pton('3.4.5.6')).port == 2) + data = data2.data + os.remove(tmp) + + assert(data.is_active == true) + local routing_table = data.routes.route + assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) + assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) + assert(routing_table:lookup_ptr(ipv4:pton('3.4.5.6')).value.port == 2) end diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 42fc3d2758..a225456c0b 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -6,14 +6,34 @@ local parse_string = require("lib.yang.parser").parse_string local schema = require("lib.yang.schema") local util = require("lib.yang.util") local value = require("lib.yang.value") +local ffi = require("ffi") +local ctable = require('lib.ctable') function normalize_id(id) return id:gsub('[^%w_]', '_') end function data_grammar_from_schema(schema) - local function struct_ctype(members) end - local function value_ctype(type) end + local function struct_ctype(members) + local member_names = {} + for k,v in pairs(members) do + if not v.ctype then return nil end + table.insert(member_names, k) + end + table.sort(member_names) + local ctype = 'struct { ' + for _,k in ipairs(member_names) do + -- Separate the array suffix off of things like "uint8_t[4]". + local head, tail = members[k].ctype:match('^([^%[]*)(.*)$') + ctype = ctype..head..' '..normalize_id(k)..tail..'; ' + end + ctype = ctype..'}' + return ctype + end + local function value_ctype(type) + -- Note that not all primitive types have ctypes. + return assert(value.types[assert(type.primitive_type)]).ctype + end local handlers = {} local function visit(node) local handler = handlers[node.kind] @@ -54,9 +74,11 @@ function data_grammar_from_schema(schema) value_ctype=struct_ctype(values)}} end function handlers.leaf(node) + local ctype + if node.default or node.mandatory then ctype=value_ctype(node.type) end return {[node.id]={type='scalar', argument_type=node.type, default=node.default, mandatory=node.mandatory, - ctype=value_ctype(node.type)}} + ctype=ctype}} end local members = visit_body(schema) return {type="struct", members=members, ctype=struct_ctype(members)} @@ -210,9 +232,11 @@ local function table_parser(keyword, keys, values, key_ctype, value_ctype) function init() return ctable.new({ key_type=key_t, value_type=value_t }) end - elseif key_t or value_t then - error('mixed table types currently unimplemented') else + -- TODO: here we should implement mixed table types if key_t or + -- value_t is non-nil, or string-keyed tables if the key is a + -- string. For the moment, fall back to the old assoc + -- implementation. function init() return make_assoc() end end local function parse1(node) From afe6195cc1a49c755ab50561068020bcf8ed9edd Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 15:05:28 +0100 Subject: [PATCH 098/631] Fix nits in yang support This adds a stub enumeration implementation. --- src/lib/yang/README.md | 4 ++-- src/lib/yang/data.lua | 2 +- src/lib/yang/value.lua | 8 ++++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index ce7c2eabdc..8c7d376f2e 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -233,7 +233,7 @@ would do: ```lua local yang = require('lib.yang.yang') -local ipv4 = require('lib.protocol.ipv4) +local ipv4 = require('lib.protocol.ipv4') local data = yang.load_data_for_schema(router_schema, conf_str) local port = data.routes.route:lookup(ipv4:pton('1.2.3.4')).port assert(port == 1) @@ -251,7 +251,7 @@ limitation that UTF-8 data is not decoded. Lua strings look like strings but really they are byte arrays. There is special support for the `ipv4-address`, `ipv4-prefix`, -`ipv6-address`, and `ipv6-address` types from `ietf-inet-types`, and +`ipv6-address`, and `ipv6-prefix` types from `ietf-inet-types`, and `mac-address` from `ietf-yang-types`. Values of these types are instead parsed to raw binary data that is compatible with the relevant parts of Snabb's `lib.protocol` facility. diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 0812c3ce24..3bee000f5d 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -71,7 +71,7 @@ local function value_parser(typ) validate = length_validator(typ.length, validate) validate = pattern_validator(typ.pattern, validate) validate = bit_validator(typ.bit, validate) - validate = enum_validator(typ.enum, validate) + validate = enum_validator(typ.enums, validate) -- TODO: union, require-instance. return function(str, k) local val = parse(str, k) diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua index 539d8f36bd..8cc30e0518 100644 --- a/src/lib/yang/value.lua +++ b/src/lib/yang/value.lua @@ -78,6 +78,14 @@ function types.string.tostring(val) return val end +types.enumeration = {} +function types.enumeration.parse(str, what) + return assert(str, 'missing value for '..what) +end +function types.enumeration.tostring(val) + return val +end + types.union = unimplemented('union') types['ipv4-address'] = { From 467e0a29144ff45d27a8552c46162da5270fe464 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 3 Nov 2016 13:33:13 +0100 Subject: [PATCH 099/631] Support cltable, string-keyed tables YANG list data with a single string key now parses to a normal string-keyed Lua table. Data with an FFI key but a Lua value is represented as a cltable. The binary representation supports all of the types. --- src/lib/yang/README.md | 2 +- src/lib/yang/binary.lua | 191 +++++++++++++++++++--------------------- src/lib/yang/data.lua | 99 ++++++++++++--------- 3 files changed, 148 insertions(+), 144 deletions(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index ce7c2eabdc..f04146ff1a 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -28,7 +28,7 @@ module snabb-simple-router { list route { key addr; leaf addr { type inet:ipv4-address; mandatory true; } - leaf port { type uint8 { range 0..11; }; mandatory true; } + leaf port { type uint8 { range 0..11; } mandatory true; } } } } diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index d7b14a20eb..baf545569b 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -78,53 +78,6 @@ local function read_string_table(stream, strtab_len) return strings end --- TODO: Implement raw structs, arrays, and tables, as follows: - --- RawStruct ::= Tag CData --- CData ::= CType CDataContents --- CType ::= StringRef of ffi type --- CDataContents ::= raw data as bytes --- --- -- CType is something like "struct { int32 a; double b; }", suitable --- -- to pass to ffi.typeof. The data itself will be aligned to the type --- -- alignment and padded on the end to next 32-bit boundary. - --- -- For RawArray, the CType indicates the type of a single item. --- RawArray ::= Tag ArrayCount CData --- CArray ::= CType CArrayContents --- CType ::= StringRef of ffi type --- CArrayContents ::= raw data as bytes - --- -- Here the raw keys or values, respectively, are collected into an --- -- array as in RawArray, and then the tagged values or keys follow. --- -- Note that the CType denotes the key or value type, not the type of --- -- the array as a whole. As before, when the table is loaded by the --- -- data plane, it will have to build a Lua table on its own. (Perhaps --- -- we could optimize this in the RawTagged case by using a ctable for --- -- the keys.) --- RawTaggedTable ::= Tag TableCount CData SubData... --- TaggedRawTable ::= Tag TableCount CData SubData... --- --- -- This is the case we are optimizing for with the binding table: here --- -- we just serialize the CTable. --- RawRawTable ::= Tag CTable --- CTable ::= KeyType ValueType CTableHeader CTableData --- KeyType ::= CType --- ValueType ::= CType --- CTableHeader ::= CTableSize CTableOccupancy CTableMaxDisplacement \ --- CTableMaxOccupancyRate CTableMinOccupancyRate --- CTableSize ::= uint32 count for number of slots in table including empty --- CTableOccupancy ::= uint32 count for number of occupied slots in table --- CTableMaxDisplacement ::= uint32 measured maximum displacement for table --- CTableMaxOccupancyRate ::= configured maximum fraction of occupied slots as double before resize --- CTableMinOccupancyRate ::= configured minimum fraction of occupied slots as double before resize --- CTableData ::= raw entries as bytes --- --- -- The number of slots in the table will be CTableSize + --- -- CTableMaxDisplacement + 1. We assume that the code that generates --- -- the compiled configuration is using the same hash function as the --- -- code that uses the compiled table. - local value_emitters = {} local function value_emitter(ctype) if value_emitters[ctype] then return value_emitters[ctype] end @@ -163,12 +116,16 @@ local function data_emitter(production) for k,_ in pairs(production.members) do table.insert(member_names, k) end table.sort(member_names) if production.ctype then - error('unimplemented') + return function(data, stream) + stream:write_stringref('cdata') + stream:write_stringref(production.ctype) + stream:write_ptr(data) + end else local emit_member = visitn(production.members) local normalize_id = data.normalize_id return function(data, stream) - stream:write_stringref('tagged-struct') + stream:write_stringref('lstruct') stream:write_uint32(table_size(data)) for _,k in ipairs(member_names) do local id = normalize_id(k) @@ -182,12 +139,18 @@ local function data_emitter(production) end function handlers.array(production) if production.ctype then - error('unimplemented') + local emit_value = value_emitter(production.ctype) + return function(data, stream) + stream:write_stringref('carray') + stream:write_stringref(production.ctype) + stream:write_uint32(#data) + stream:write_array(data, ffi.typeof(production.ctype), #data) + end else local emit_tagged_value = visit1( {type='scalar', argument_type=production.element_type}) return function(data, stream) - stream:write_stringref('tagged-array') + stream:write_stringref('larray') stream:write_uint32(#data) for i=1,#data do emit_tagged_value(data[i], stream) end end @@ -201,22 +164,41 @@ local function data_emitter(production) stream:write_stringref(production.value_ctype) data:save(stream) end - else - -- TODO: here we should implement mixed table types if key_t or - -- value_t is non-nil, or string-keyed tables if the key is a - -- string. For the moment, fall back to the old assoc - -- implementation. - -- if production.key_ctype then error('unimplemented') end - -- if production.value_ctype then error('unimplemented') end - local emit_key = visit1({type='struct', members=production.keys}) + elseif production.string_key then + local emit_value = visit1({type='struct', members=production.values, + ctype=production.value_ctype}) + -- FIXME: sctable if production.value_ctype? + return function(data, stream) + -- A string-keyed table is the same as a tagged struct. + stream:write_stringref('lstruct') + stream:write_uint32(table_size(data)) + for k,v in pairs(data) do + stream:write_stringref(k) + emit_value(v, stream) + end + end + elseif production.key_ctype then + local emit_keys = visit1({type='table', key_ctype=production.key_ctype, + value_ctype='uint32_t'}) local emit_value = visit1({type='struct', members=production.values}) return function(data, stream) - stream:write_stringref('tagged-table') - stream:write_uint32(#data) - -- FIXME: relies on data being an assoc - for _,v in ipairs(data) do - emit_key(v.key, stream) - emit_value(v.value, stream) + stream:write_stringref('cltable') + emit_keys(data.keys) + stream:write_uint32(#data.values) + for i=1,#data.values do emit_value(data.values[i], stream) end + end + else + local emit_key = visit1({type='struct', members=production.keys, + ctype=production.key_ctype}) + local emit_value = visit1({type='struct', members=production.values, + ctype=production.value_ctype}) + -- FIXME: lctable if production.value_ctype? + return function(data, stream) + stream:write_stringref('lltable') + stream:write_uint32(table_count(data)) + for k,v in pairs(data) do + emit_key(k, stream) + emit_value(v, stream) end end end @@ -226,7 +208,7 @@ local function data_emitter(production) -- FIXME: needs a case for unions if primitive_type == 'string' then return function(data, stream) - stream:write_stringref('tagged-string') + stream:write_stringref('stringref') stream:write_stringref(data) end else @@ -290,12 +272,18 @@ local function read_compiled_data(stream, strtab) local function read_string() return assert(strtab[stream:read_uint32()]) end + local ctypes = {} + local function scalar_type(ctype) + if not ctypes[ctype] then ctypes[ctype] = ffi.typeof(ctype) end + return ctypes[ctype] + end + local readers = {} local function read1() local tag = read_string() return assert(readers[tag], tag)() end - readers['tagged-struct'] = function () + function readers.lstruct() local ret = {} for i=1,stream:read_uint32() do local k = read_string() @@ -303,12 +291,28 @@ local function read_compiled_data(stream, strtab) end return ret end - readers['tagged-array'] = function () + function readers.carray() + local ctype = scalar_type(read_string()) + return stream:read_array(ctype, stream:read_uint32()) + end + function readers.larray() local ret = {} for i=1,stream:read_uint32() do table.insert(ret, read1()) end return ret end - readers['tagged-table'] = function () + function readers.ctable() + local key_ctype = read_string() + local value_ctype = read_string() + local key_t, value_t = ffi.typeof(key_ctype), ffi.typeof(value_ctype) + return ctable.load(stream, {key_type=key_t, value_type=value_t}) + end + function readers.cltable() + local keys = read1() + local values = {} + for i=1,stream:read_uint32() do table.insert(values, read1()) end + return cltable.build(keys, values) + end + function readers.lltable() local ret = data.make_assoc() for i=1,stream:read_uint32() do local k = read1() @@ -316,21 +320,10 @@ local function read_compiled_data(stream, strtab) end return ret end - readers['ctable'] = function () - local key_ctype = read_string() - local value_ctype = read_string() - local key_t, value_t = ffi.typeof(key_ctype), ffi.typeof(value_ctype) - return ctable.load(stream, {key_type=key_t, value_type=value_t}) - end - readers['tagged-string'] = function () + function readers.stringref() return read_string() end - local ctypes = {} - local function scalar_type(ctype) - if not ctypes[ctype] then ctypes[ctype] = ffi.typeof(ctype) end - return ctypes[ctype] - end - readers['cdata'] = function () + function readers.cdata() local ctype = scalar_type(read_string()) return stream:read_ptr(ctype)[0] end @@ -365,6 +358,7 @@ function load_compiled_data_file(filename) end function selftest() + print('selfcheck: lib.yang.binary') local test_schema = schema.load_schema([[module snabb-simple-router { namespace snabb:simple-router; prefix simple-router; @@ -393,23 +387,20 @@ function selftest() local ipv4 = require('lib.protocol.ipv4') - assert(data.is_active == true) - local routing_table = data.routes.route - assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) - assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) - assert(routing_table:lookup_ptr(ipv4:pton('3.4.5.6')).value.port == 2) + for i=1,3 do + assert(data.is_active == true) + local routing_table = data.routes.route + assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) + assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) + assert(routing_table:lookup_ptr(ipv4:pton('3.4.5.6')).value.port == 2) - local tmp = os.tmpname() - compile_data_for_schema(test_schema, data, tmp) - local data2 = load_compiled_data_file(tmp) - assert(data2.schema_name == 'snabb-simple-router') - assert(data2.revision_date == '') - data = data2.data - os.remove(tmp) - - assert(data.is_active == true) - local routing_table = data.routes.route - assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) - assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) - assert(routing_table:lookup_ptr(ipv4:pton('3.4.5.6')).value.port == 2) + local tmp = os.tmpname() + compile_data_for_schema(test_schema, data, tmp) + local data2 = load_compiled_data_file(tmp) + assert(data2.schema_name == 'snabb-simple-router') + assert(data2.revision_date == '') + data = data2.data + os.remove(tmp) + end + print('selfcheck: ok') end diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index a225456c0b..7f67afcf8e 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -8,11 +8,26 @@ local util = require("lib.yang.util") local value = require("lib.yang.value") local ffi = require("ffi") local ctable = require('lib.ctable') +local cltable = require('lib.cltable') function normalize_id(id) return id:gsub('[^%w_]', '_') end +-- If a "list" node has one key that is string-valued, we will represent +-- instances of that node as normal Lua tables where the key is the +-- table key and the value does not contain the key. +local function table_string_key(keys) + local string_key = nil + for k,v in pairs(keys) do + if v.type ~= 'scalar' then return nil end + if v.argument_type.primitive_type ~= 'string' then return nil end + if string_key ~= nil then return nil end + string_key = k + end + return string_key +end + function data_grammar_from_schema(schema) local function struct_ctype(members) local member_names = {} @@ -70,6 +85,7 @@ function data_grammar_from_schema(schema) if not keys[k] then values[k] = v end end return {[node.id]={type='table', keys=keys, values=values, + string_key=table_string_key(keys), key_ctype=struct_ctype(keys), value_ctype=struct_ctype(values)}} end @@ -198,29 +214,28 @@ local function scalar_parser(keyword, argument_type, default, mandatory) return {init=init, parse=parse, finish=finish} end --- Simple temporary associative array until we get the various Table --- kinds working. -function make_assoc() - local assoc = {} - function assoc:get_entry(k) - assert(type(k) ~= 'table', 'multi-key lookup unimplemented') - for _,entry in ipairs(self) do - for _,v in pairs(entry.key) do - if v == k then return entry end - end - end - error('not found: '..k) - end - function assoc:get_key(k) return self:get_entry(k).key end - function assoc:get_value(k) return self:get_entry(k).value end - function assoc:add(k, v, check) - if check then assert(not self:get_entry(k)) end - table.insert(self, {key=k, value=v}) - end - return assoc +local function ctable_builder(key_t, value_t) + local res = ctable.new({ key_type=key_t, value_type=value_t }) + local builder = {} + function builder:add(key, value) res:add(key, value) end + function builder:finish() return res end + return builder +end + +local function string_keyed_table_builder(string_key) + local res = {} + local builder = {} + function builder:add(key, value) + local str = assert(key[string_key]) + assert(res[str] == nil, 'duplicate key: '..str) + res[str] = value + end + function builder:finish() return res end + return builder end -local function table_parser(keyword, keys, values, key_ctype, value_ctype) +local function table_parser(keyword, keys, values, string_key, key_ctype, + value_ctype) local members = {} for k,v in pairs(keys) do members[k] = v end for k,v in pairs(values) do members[k] = v end @@ -229,15 +244,16 @@ local function table_parser(keyword, keys, values, key_ctype, value_ctype) local value_t = value_ctype and ffi.typeof(value_ctype) local init if key_t and value_t then - function init() - return ctable.new({ key_type=key_t, value_type=value_t }) - end + function init() return ctable_builder(key_t, value_t) end + elseif string_key then + function init() return string_keyed_table_builder(string_key) end else - -- TODO: here we should implement mixed table types if key_t or - -- value_t is non-nil, or string-keyed tables if the key is a - -- string. For the moment, fall back to the old assoc - -- implementation. - function init() return make_assoc() end + -- TODO: here we should implement a cktable if key_t is non-nil. + -- Probably we should error if the key is a generic Lua table + -- though, as we don't have a good data structure to map generic + -- Lua tables to Lua tables. For the moment, fall back to the old + -- assoc implementation. + error('List with non-FFI, non-string key unimplemented') end local function parse1(node) assert_compound(node, keyword) @@ -256,7 +272,7 @@ local function table_parser(keyword, keys, values, key_ctype, value_ctype) return assoc end local function finish(assoc) - return assoc + return assoc:finish() end return {init=init, parse=parse, finish=finish} end @@ -286,8 +302,8 @@ function data_parser_from_grammar(production) end function handlers.table(keyword, production) local keys, values = visitn(production.keys), visitn(production.values) - return table_parser(keyword, keys, values, production.key_ctype, - production.value_ctype) + return table_parser(keyword, keys, values, production.string_key, + production.key_ctype, production.value_ctype) end function handlers.scalar(keyword, production) return scalar_parser(keyword, production.argument_type, @@ -311,6 +327,7 @@ function load_data_for_schema_by_name(schema_name, str, filename) end function selftest() + print('selfcheck: lib.yang.data') local test_schema = schema.load_schema([[module fruit { namespace "urn:testing:fruit"; prefix "fruit"; @@ -349,16 +366,12 @@ function selftest() ]]) assert(data.fruit_bowl.description == 'ohai') local contents = data.fruit_bowl.contents - assert(contents:get_entry('foo').key.name == 'foo') - assert(contents:get_entry('foo').value.score == 7) - assert(contents:get_key('foo').name == 'foo') - assert(contents:get_value('foo').score == 7) - assert(contents:get_value('foo').tree_grown == nil) - assert(contents:get_key('bar').name == 'bar') - assert(contents:get_value('bar').score == 8) - assert(contents:get_value('bar').tree_grown == nil) - assert(contents:get_key('baz').name == 'baz') - assert(contents:get_value('baz').score == 9) - assert(contents:get_value('baz').tree_grown == true) + assert(contents.foo.score == 7) + assert(contents.foo.tree_grown == nil) + assert(contents.bar.score == 8) + assert(contents.bar.tree_grown == nil) + assert(contents.baz.score == 9) + assert(contents.baz.tree_grown == true) assert(require('lib.protocol.ipv4'):ntop(data.addr) == '1.2.3.4') + print('selfcheck: ok') end From 69ca0cf30bfd027bdc654f757ae351f577e564a2 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 3 Nov 2016 14:55:08 +0100 Subject: [PATCH 100/631] Fix bugs in raw leaf-lists Also update documentation a bit. --- src/lib/yang/README.md | 61 ++++++++++++++++++++++++++++++++--------- src/lib/yang/binary.lua | 13 +++++++-- src/lib/yang/data.lua | 10 +++++-- src/lib/yang/util.lua | 23 ++++++++++++++++ 4 files changed, 89 insertions(+), 18 deletions(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index 32ec01aa9f..9165c7e7a8 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -225,17 +225,18 @@ The `routes` container is just another table of the same kind. corresponding nodes in the configuration syntax, and corresponding sub-tables in the result configuration objects.) -Inside the `routes` container is the `route` list, which is also -represented as a table. Recall that in YANG, `list` types are really -key-value associations, so the `route` table has a `:lookup` method to -get its sub-items. Therefore to get the port for address 1.2.3.4, you -would do: +Inside the `routes` container is the `route` list, which is represented +as an associative array. The particular representation for the +associative array depends on characteristics of the `list` type; see +below for details. In this case the `route` list compiles to a +[`ctable`](../README.ctable.md). Therefore to get the port for address +1.2.3.4, you would do: ```lua local yang = require('lib.yang.yang') local ipv4 = require('lib.protocol.ipv4') local data = yang.load_data_for_schema(router_schema, conf_str) -local port = data.routes.route:lookup(ipv4:pton('1.2.3.4')).port +local port = data.routes.route:lookup_ptr(ipv4:pton('1.2.3.4')).value.port assert(port == 1) ``` @@ -256,15 +257,49 @@ There is special support for the `ipv4-address`, `ipv4-prefix`, parsed to raw binary data that is compatible with the relevant parts of Snabb's `lib.protocol` facility. -Returning to compound configuration data types, configuration for -`leaf-list` schema nodes are represented as normal arrays, whose values -are instances of the leaf types. +Let us return to the representation of compound configurations, like +`list` instances. A compound configuration whose shape is *fixed* is +compiled to raw FFI data. A configuration's shape is determined by its +schema. A schema node whose data will be fixed is either a leaf whose +type is numeric or boolean and which is either mandatory or has a +default value, or a container (`leaf-list`, `container` with presence, +or `list`) whose elements are all themselves fixed. + +In practice this means that a fixed `container` with presence will be +compiled to an FFI `struct` type. This is mostly transparent from the +user perspective, as in LuaJIT you access struct members by name in the +same way as for normal Lua tables. + +A fixed `leaf-list` will be compiled to an FFI array of its element +type, but on the Lua side is given the normal 1-based indexing and +support for the `#len` length operator via a wrapper. A non-fixed +`leaf-list` is just a Lua array (a table with indexes starting from 1). + +Instances of `list` nodes can have one of several representations. +(Recall that in YANG, `list` is not a list in the sense that we normally +think of it in programming languages, but rather is a kind of hash map.) + +If there is only one key leaf, and that leaf has a string type, then a +configuration list is represented as a normal Lua table whose keys are +the key strings, and whose values are Lua structures holding the leaf +values, as in containers. (In fact, it could be that the value of a +string-key struct is represented as a C struct, as in raw containers.) + +If all key and value types are fixed, then a `list` configuration +compiles to an efficient [`ctable`](../README.ctable.md). + +If all keys are fixed but values are not, then a `list` configuration +compiles to a [`cltable`](../README.cltable.md). + +Otherwise, a `list` configuration compiles to a Lua table whose keys are +Lua tables containing the keys. This sounds good on the surface but +really it's a pain, because you can't simply look up a value in the +table like `foo[{key1=42,key2=50}]`, because lookup in such a table is +by identity and not be value. Oh well. You can still do `for k,v in +pairs(foo)`, which is often good enough in this case. Note that there are a number of value types that are not implemented, -including some important ones like `union`, and the `list` type -representation needs further optimization. We aim to compile `list` -values directly to `ctable` instances where possible. Patches are -welcome :) +including some important ones like `union`. — Function **load_data_for_schema_by_name** *schema_name* *name* *filename* diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index baf545569b..cb19d2ed44 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -144,7 +144,7 @@ local function data_emitter(production) stream:write_stringref('carray') stream:write_stringref(production.ctype) stream:write_uint32(#data) - stream:write_array(data, ffi.typeof(production.ctype), #data) + stream:write_array(data.ptr, ffi.typeof(production.ctype), #data) end else local emit_tagged_value = visit1( @@ -293,7 +293,8 @@ local function read_compiled_data(stream, strtab) end function readers.carray() local ctype = scalar_type(read_string()) - return stream:read_array(ctype, stream:read_uint32()) + local count = stream:read_uint32() + return util.ffi_array(stream:read_array(ctype, count), ctype, count) end function readers.larray() local ret = {} @@ -367,6 +368,7 @@ function selftest() leaf is-active { type boolean; default true; } + leaf-list integers { type uint32; } container routes { presence true; list route { @@ -378,6 +380,9 @@ function selftest() }]]) local data = data.load_data_for_schema(test_schema, [[ is-active true; + integers 1; + integers 2; + integers 0xffffffff; routes { route { addr 1.2.3.4; port 1; } route { addr 2.3.4.5; port 10; } @@ -389,6 +394,10 @@ function selftest() for i=1,3 do assert(data.is_active == true) + assert(#data.integers == 3) + assert(data.integers[1] == 1) + assert(data.integers[2] == 2) + assert(data.integers[3] == 0xffffffff) local routing_table = data.routes.route assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 42174f05f0..4faf137aea 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -181,16 +181,20 @@ local function array_parser(keyword, element_type, ctype) local parsev = value_parser(element_type) local function parse1(node) assert_scalar(node, keyword) - return parsev(node.argument, k) + return parsev(node.argument, keyword) end local function parse(node, out) table.insert(out, parse1(node)) return out end - local array_t = ctype and ffi.typeof('$[?]', ffi.typeof(ctype)) + local elt_t = ctype and ffi.typeof(ctype) + local array_t = ctype and ffi.typeof('$[?]', ffi.typeof(elt_t)) local function finish(out) -- FIXME check min-elements - if array_t then return array_t(out) else return out end + if array_t then + out = util.ffi_array(array_t(#out, out), elt_t) + end + return out end return {init=init, parse=parse, finish=finish} end diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 61391c307e..955678c370 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -47,6 +47,29 @@ function tointeger(str, what, min, max) return res end +function ffi_array(ptr, elt_t, count) + local mt = {} + local size = count or ffi.sizeof(ptr)/ffi.sizeof(elt_t) + function mt:__len() return size end + function mt:__index(idx) + assert(1 <= idx and idx <= size) + return ptr[idx-1] + end + function mt:__setindex(idx, val) + assert(1 <= idx and idx <= size) + ptr[idx-1] = val + end + function mt:__ipairs() + local idx = -1 + return function() + idx = idx + 1 + if idx >= size then return end + return idx+1, ptr[idx] + end + end + return ffi.metatype(ffi.typeof('struct { $* ptr; }', elt_t), mt)(ptr) +end + function selftest() assert(tointeger('0') == 0) assert(tointeger('-0') == 0) From 2c0fab2bedb344701de3600f78a988a725509ddf Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 3 Nov 2016 15:43:24 +0100 Subject: [PATCH 101/631] Update lib.yang.yang Also update README to match current status. --- src/lib/yang/README.md | 45 ++++++++++++++++++++++++++++++++++++++++-- src/lib/yang/yang.lua | 6 ++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index 9165c7e7a8..0335cc356f 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -144,8 +144,19 @@ hand. #### Compiled configurations -[TODO] We will support compiling configurations to an efficient binary -representation that can be loaded without validation. +Loading a schema and using it to parse a data file can be a bit +expensive, especially if the data file includes a large routing table or +other big structure. It can be useful to pay for this this parsing and +validation cost "offline", without interrupting a running data plane. + +For this reason, Snabb support compiling configurations to binary data. +A data plane can load a compiled configuration without any validation, +very cheaply. Users can explicitly call the `compile_data_for_schema` +or `compile_data_for_schema_by_name` functions. Support is planned also +for automatic compilation and of source configuration files as well, so +that the user can just edit configurations as text and still take +advantage of the speedy binary configuration loads when nothing has +changed. #### Querying and updating configurations @@ -305,3 +316,33 @@ including some important ones like `union`. Like `load_data_for_schema`, but identifying the schema by name instead of by value, as in `load_schema_by_name`. + +— Function **compile_data_for_schema** *schema* *data* *filename* *mtime* + +Compile *data*, using a compiler generated for *schema*, and write out +the result to the file named *filename*. *mtime*, if given, should be a +table with `secs` and `nsecs` keys indicating the modification time of +the source file. This information will be serialized in the compiled +file, and may be used when loading the file to determine whether the +configuration is up to date. + +— Function **compile_data_for_schema_by_name** *schema_name* *data* *filename* *mtime* + +Like `compile_data_for_schema_by_name`, but identifying the schema by +name instead of by value, as in `load_schema_by_name`. + +— Function **load_compiled_data_file** *filename* + +Load the compiled data file at *filename*. If the file is not a +compiled YANG configuration, an error will be signalled. The return +value will be table containing four keys: + + * `schema_name`: The name of the schema for which this file was + compiled. + * `revision_date`: The revision date of the schema for which this file + was compiled, or the empty string (`''`) if unknown. + * `source_mtime`: An `mtime` table, as for `compile_data_for_schema`. + If no mtime was written into the file, both `secs` and `nsecs` will + be zero. + * `data`: The configuration data, in the same format as returned by + `load_data_for_schema`. diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 8837ab9f1b..558928cfc2 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -3,6 +3,7 @@ module(..., package.seeall) local schema = require("lib.yang.schema") local data = require("lib.yang.data") +local binary = require("lib.yang.binary") load_schema = schema.load_schema load_schema_file = schema.load_schema_file @@ -11,5 +12,10 @@ load_schema_by_name = schema.load_schema_by_name load_data_for_schema = data.load_data_for_schema load_data_for_schema_by_name = data.load_data_for_schema_by_name +compile_data_for_schema = binary.compile_data_for_schema +compile_data_for_schema_by_name = binary.compile_data_for_schema_by_name + +load_compiled_data_file = binary.load_compiled_data_file + function selftest() end From 9dfb3c89caacbd9f7fa8fca07af366c1f58634bd Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 3 Nov 2016 15:55:27 +0100 Subject: [PATCH 102/631] Minor cleanup of comment to re-trigger CI bot. --- src/core/app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/app.lua b/src/core/app.lua index 88e3b1b7cb..03b9fe8d2c 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -26,7 +26,7 @@ local use_restart = false test_skipped_code = 43 --- Set the directories for the named programs. +-- Set the directory for the named programs. named_program_root = shm.root .. "/" .. "by-name" -- The set of all active apps and links in the system. From dd5905f1bc71bc7533e5b3a4e939cd1fb41795a4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 3 Nov 2016 17:01:56 +0100 Subject: [PATCH 103/631] Add auto-compilation interface The new lib.yang.yang.load_configuration procedure will automatically compile source configurations to binary files, and will automatically take advantage of compiled binary files if possible. --- src/lib/yang/README.md | 30 ++++++++++- src/lib/yang/binary.lua | 6 +++ src/lib/yang/stream.lua | 3 ++ src/lib/yang/yang.lua | 114 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index 0335cc356f..e577d6e8b2 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -187,7 +187,35 @@ partial) state data. #### API reference The public entry point to the YANG library is the `lib.yang.yang` -module, which exports the following bindings: +module, which exports the following bindings. Note that unless you have +special needs, probably the only one you want to use is +`load_configuration`. + +— Function **load_configuration** *filename* *parameters* + +Load a configuration from disk. If *filename* is a compiled +configuration, load it directly. Otherwise it must be a source file. +In that case, try to load a corresponding compiled file instead if +possible. If all that fails, actually parse the source configuration, +and try to residualize a corresponding compiled file so that we won't +have to go through the whole thing next time. + +*parameters* is a table of key/value pairs. The following key is +required: + + * `schema_name`: The name of the YANG schema that describes the + configuration. This is the name that appears as the *id* in `module + id { ... }` in the schema. + +Optional entries that may be present in the *parameters* table include: + + * `verbose`: Set to true to print verbose information about which files + are being loaded and compiled. + * `revision_date`: If set, assert that the loaded configuration was + built against this particular schema revision date. + +For more information on the format of the returned value, see the +documentation below for `load_data_for_schema`. — Function **load_schema** *src* *filename* diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index cb19d2ed44..32b47eee0d 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -331,6 +331,12 @@ local function read_compiled_data(stream, strtab) return read1() end +function has_magic(stream) + local success, header = pcall(stream.read_ptr, stream, header_t) + stream:seek(0) + return success and ffi.string(header.magic, ffi.sizeof(header.magic)) == MAGIC +end + function load_compiled_data(stream) local uint32_t = ffi.typeof('uint32_t') function stream:read_uint32() diff --git a/src/lib/yang/stream.lua b/src/lib/yang/stream.lua index 14f3925ac7..2800b7e3a7 100644 --- a/src/lib/yang/stream.lua +++ b/src/lib/yang/stream.lua @@ -142,6 +142,9 @@ function open_input_byte_stream(filename) function ret:read_char() return ffi.string(ret:read(1), 1) end + function ret:read_string() + return ffi.string(ret:read(size - pos), size - pos) + end function ret:as_text_stream(len) local end_pos = size if len then end_pos = pos + len end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 558928cfc2..4c9676f6c6 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -1,9 +1,11 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) +local lib = require("core.lib") local schema = require("lib.yang.schema") local data = require("lib.yang.data") local binary = require("lib.yang.binary") +local stream = require("lib.yang.stream") load_schema = schema.load_schema load_schema_file = schema.load_schema_file @@ -17,5 +19,117 @@ compile_data_for_schema_by_name = binary.compile_data_for_schema_by_name load_compiled_data_file = binary.load_compiled_data_file +local params = { + verbose = {}, + schema_name = {required=true}, + revision_date = {}, +} + +-- Load the configuration from FILENAME. If it's compiled, load it +-- directly. Otherwise if it's source, then try to load a corresponding +-- compiled file instead if possible. If all that fails, actually parse +-- the source configuration, and try to residualize a corresponding +-- compiled file so that we won't have to go through the whole thing +-- next time. +function load_configuration(filename, opts) + opts = lib.parse(opts, params) + + function maybe(f, ...) + local function catch(success, ...) + if success then return ... end + end + return catch(pcall(f, ...)) + end + local function err_msg(msg, ...) + return string.format('%s: '..msg, filename, ...) + end + local function err(msg, ...) error(err_msg(msg, ...)) end + local function log(msg, ...) + io.stderr:write(err_msg(msg, ...)..'\n') + io.stderr:flush() + end + local function assert(exp, msg, ...) + if exp then return exp else err(msg, ...) end + end + local function expect(expected, got, what) + assert(expected == got, 'expected %s %s, but got %s', what, expected, got) + end + + local function is_fresh(expected, got) + end + local function load_compiled(stream, source_mtime) + local compiled = binary.load_compiled_data(stream) + if opts.schema_name then + expect(opts.schema_name, compiled.schema_name, 'schema name') + end + if opts.revision_date then + expect(opts.revision_date, compiled.revision_date, + 'schema revision date') + end + if source_mtime == nil or + (source_mtime.sec == compiled.source_mtime.sec and + source_mtime.nsec == compiled.source_mtime.nsec) then + return compiled.data + end + end + + local source = stream.open_input_byte_stream(filename) + if binary.has_magic(source) then return load_compiled(source) end + + -- If the file doesn't have the magic, assume it's a source file. + -- First, see if we compiled it previously and saved a compiled file + -- in a well-known place. + local compiled_filename = filename:gsub("%.txt$", "")..'.o' + local source_mtime = {sec=source.mtime_sec, nsec=source.mtime_nsec} + local compiled_stream = maybe(stream.open_input_byte_stream, + compiled_filename) + if compiled_stream then + if binary.has_magic(compiled_stream) then + log('loading compiled configuration from %s', compiled_filename) + local conf = load_compiled(compiled_stream, source_mtime) + if conf then + log('compiled configuration %s is up to date.', compiled_filename) + return conf + end + log('compiled configuration %s is out of date; recompiling.', + compiled_filename) + end + compiled_stream:close() + end + + -- Load and compile it. + local source_str = source:read_string() + source:close() + log('loading source configuration from %s', filename) + local conf = load_data_for_schema_by_name(opts.schema_name, source_str, + filename) + + -- Save it, if we can. + local success, err = pcall(binary.compile_data_for_schema_by_name, + opts.schema_name, conf, compiled_filename, + source_mtime) + if not success then + log('error saving compiled configuration %s: %s', compiled_filename, err) + end + + -- Done. + return conf +end + function selftest() + print('selftest: lib.yang.yang') + local tmp = os.tmpname() + do + local file = io.open(tmp, 'w') + -- ietf-yang-types only defines types. FIXME: use a schema that + -- actually defines some data nodes. + file:write('/* nothing */') + file:close() + end + load_configuration(tmp, {schema_name='ietf-yang-types'}) + load_configuration(tmp, {schema_name='ietf-yang-types'}) + os.remove(tmp) + load_configuration(tmp..'.o', {schema_name='ietf-yang-types'}) + os.remove(tmp..'.o') + print('selftest: ok') end From 93ca00c3e022c5ca272adc542808927b74834a82 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 10:25:02 +0100 Subject: [PATCH 104/631] Add support to serialize a YANG config as text --- src/lib/yang/README.md | 11 +++ src/lib/yang/data.lua | 167 ++++++++++++++++++++++++++++++++++++++--- src/lib/yang/yang.lua | 3 + 3 files changed, 171 insertions(+), 10 deletions(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index e577d6e8b2..7e8fe2acde 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -345,6 +345,17 @@ including some important ones like `union`. Like `load_data_for_schema`, but identifying the schema by name instead of by value, as in `load_schema_by_name`. +— Function **print_data_for_schema** *schema* *data* *file* + +Serialize the configuration *data* as text via repeated calls to the +`write` method of *file*. At the end, the `flush` method is called on +*file*. *schema* is the schema that describes *data*. + +— Function **print_data_for_schema_by_name** *schema_name* *name* *filename* + +Like `print_data_for_schema`, but identifying the schema by name instead +of by value, as in `load_schema_by_name`. + — Function **compile_data_for_schema** *schema* *data* *filename* *mtime* Compile *data*, using a compiler generated for *schema*, and write out diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 4faf137aea..6f2e7eb6dd 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -213,7 +213,7 @@ local function scalar_parser(keyword, argument_type, default, mandatory) local function finish(out) if out ~= nil then return out end if default then return parse1(default) end - if mandatory then error('missing scalar value: '..k) end + if mandatory then error('missing scalar value: '..keyword) end end return {init=init, parse=parse, finish=finish} end @@ -330,6 +330,142 @@ function load_data_for_schema_by_name(schema_name, str, filename) return load_data_for_schema(schema, str, filename) end +local function encode_yang_string(str) + if str:match("^[^%s;{}\"'/]*$") then return str end + error('yang string escaping unimplemented: '..str) +end + +local value_printers = {} +local function value_printer(typ) + local prim = typ.primitive_type + if value_printers[prim] then return value_printers[prim] end + local tostring = assert(value.types[prim], prim).tostring + local function print(val, file) + file:write(encode_yang_string(tostring(val))) + end + value_printers[prim] = print + return print +end + +local function data_printer_from_grammar(production) + local handlers = {} + local function printer(keyword, production) + return assert(handlers[production.type])(keyword, production) + end + local function print_string(str, file) + file:write(encode_yang_string(str)) + end + local function print_keyword(k, file, indent) + file:write(indent) + print_string(k, file) + file:write(' ') + end + local function body_printer(productions, order) + if not order then + order = {} + for k,_ in pairs(productions) do table.insert(order, k) end + table.sort(order) + end + local printers = {} + for keyword,production in pairs(productions) do + printers[keyword] = printer(keyword, production) + end + return function(data, file, indent) + for _,k in ipairs(order) do + local v = data[normalize_id(k)] + if v ~= nil then printers[k](v, file, indent) end + end + end + end + function handlers.struct(keyword, production) + local print_body = body_printer(production.members) + return function(data, file, indent) + print_keyword(keyword, file, indent) + file:write('{\n') + print_body(data, file, indent..' ') + file:write(indent..'}\n') + end + end + function handlers.array(keyword, production) + local print_value = value_printer(production.element_type) + return function(data, file, indent) + for _,v in ipairs(data) do + print_keyword(keyword, file, indent) + print_value(v, file) + file:write(';\n') + end + end + end + function handlers.table(keyword, production) + local key_order, value_order = {}, {} + for k,_ in pairs(production.keys) do table.insert(key_order, k) end + for k,_ in pairs(production.values) do table.insert(value_order, k) end + table.sort(key_order) + table.sort(value_order) + local print_key = body_printer(production.keys, key_order) + local print_value = body_printer(production.values, value_order) + if production.key_ctype and production.value_ctype then + return function(data, file, indent) + for entry in data:iterate() do + print_keyword(keyword, file, indent) + file:write('{\n') + print_key(entry.key, file, indent..' ') + print_value(entry.value, file, indent..' ') + file:write(indent..'}\n') + end + end + elseif production.string_key then + local id = normalize_id(production.string_key) + return function(data, file, indent) + for key, value in pairs(data) do + print_keyword(keyword, file, indent) + file:write('{\n') + print_key({[id]=key}, file, indent..' ') + print_value(value, file, indent..' ') + file:write(indent..'}\n') + end + end + else + return function(data, file, indent) + for key, value in pairs(data) do + print_keyword(keyword, file, indent) + file:write('{\n') + print_key(key, file, indent..' ') + print_value(value, file, indent..' ') + file:write(indent..'}\n') + end + end + end + end + function handlers.scalar(keyword, production) + local print_value = value_printer(production.argument_type) + return function(data, file, indent) + print_keyword(keyword, file, indent) + print_value(data, file) + file:write(';\n') + end + end + + local top_printer = body_printer(production.members) + return function(data, file) + top_printer(data, file, '') + file:flush() + end +end + +function data_printer_from_schema(schema) + return data_printer_from_grammar(data_grammar_from_schema(schema)) +end + +function print_data_for_schema(schema, data, file) + return data_printer_from_schema(schema)(data, file) +end + +function print_data_for_schema_by_name(schema_name, data, file) + local schema = schema.load_schema_by_name(schema_name) + return print_data_for_schema(schema, data, file) +end + function selftest() print('selfcheck: lib.yang.data') local test_schema = schema.load_schema([[module fruit { @@ -368,14 +504,25 @@ function selftest() } addr 1.2.3.4; ]]) - assert(data.fruit_bowl.description == 'ohai') - local contents = data.fruit_bowl.contents - assert(contents.foo.score == 7) - assert(contents.foo.tree_grown == nil) - assert(contents.bar.score == 8) - assert(contents.bar.tree_grown == nil) - assert(contents.baz.score == 9) - assert(contents.baz.tree_grown == true) - assert(require('lib.protocol.ipv4'):ntop(data.addr) == '1.2.3.4') + for i =1,2 do + assert(data.fruit_bowl.description == 'ohai') + local contents = data.fruit_bowl.contents + assert(contents.foo.score == 7) + assert(contents.foo.tree_grown == nil) + assert(contents.bar.score == 8) + assert(contents.bar.tree_grown == nil) + assert(contents.baz.score == 9) + assert(contents.baz.tree_grown == true) + assert(require('lib.protocol.ipv4'):ntop(data.addr) == '1.2.3.4') + + local tmp = os.tmpname() + local file = io.open(tmp, 'w') + print_data_for_schema(test_schema, data, file) + file:close() + local file = io.open(tmp, 'r') + local data = load_data_for_schema(test_schema, file:read('*a'), tmp) + file:close() + os.remove(tmp) + end print('selfcheck: ok') end diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 4c9676f6c6..7645c80aa9 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -14,6 +14,9 @@ load_schema_by_name = schema.load_schema_by_name load_data_for_schema = data.load_data_for_schema load_data_for_schema_by_name = data.load_data_for_schema_by_name +print_data_for_schema = data.print_data_for_schema +print_data_for_schema_by_name = data.print_data_for_schema_by_name + compile_data_for_schema = binary.compile_data_for_schema compile_data_for_schema_by_name = binary.compile_data_for_schema_by_name From df8dbf90cfdbf2e46f86c0ffbbace2ad9339d081 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 10:34:28 +0100 Subject: [PATCH 105/631] Move snabb-softwire.yang to lib/yang --- src/{apps/lwaftr => lib/yang}/snabb-softwire.yang | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{apps/lwaftr => lib/yang}/snabb-softwire.yang (100%) diff --git a/src/apps/lwaftr/snabb-softwire.yang b/src/lib/yang/snabb-softwire.yang similarity index 100% rename from src/apps/lwaftr/snabb-softwire.yang rename to src/lib/yang/snabb-softwire.yang From 391b766cac3d4adfb7dff312b489289b41618477 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 10:38:16 +0100 Subject: [PATCH 106/631] Rename to snabb-softwire-v1.yang RFC 6020 section 10 constrains the kinds of changes that additional YANG schema revisions can make. As we do anticipate changes after we publish this schema, rename the module to explicitly include a version name. --- .../yang/{snabb-softwire.yang => snabb-softwire-v1.yang} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename src/lib/yang/{snabb-softwire.yang => snabb-softwire-v1.yang} (98%) diff --git a/src/lib/yang/snabb-softwire.yang b/src/lib/yang/snabb-softwire-v1.yang similarity index 98% rename from src/lib/yang/snabb-softwire.yang rename to src/lib/yang/snabb-softwire-v1.yang index 6c98f52939..06cd90776f 100644 --- a/src/lib/yang/snabb-softwire.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -1,6 +1,6 @@ -module snabb-softwire { - namespace "snabb:lwaftr"; - prefix "softwire"; +module snabb-softwire-v1 { + namespace snabb:lwaftr; + prefix softwire; import ietf-inet-types {prefix inet; } @@ -8,7 +8,7 @@ module snabb-softwire { contact "Jessica Tallon "; description "Configuration for the Snabb Switch lwAFTR."; - revision 2016-08-01 { + revision 2016-11-04 { description "Initial revision."; reference "-00"; } From da57ac427614f0a585e2c2378968347bebaabbce Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 10:42:03 +0100 Subject: [PATCH 107/631] Hyphen-case rather than CamelCase in snabb-softwire-v1 --- src/lib/yang/snabb-softwire-v1.yang | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 06cd90776f..e825e77a52 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -13,7 +13,7 @@ module snabb-softwire-v1 { reference "-00"; } - typedef PacketPolicy { + typedef packet-policy { type enumeration { enum allow { value 0; @@ -28,14 +28,14 @@ module snabb-softwire-v1 { them."; } - typedef PositiveNumber { + typedef positive-uint32 { type uint32 { range "1.. 2147483647"; } description "This is a non-negative integer value which is 1 or above."; } - typedef VlanTag { + typedef vlan-tag { type uint16 { range "0..4095"; } @@ -145,7 +145,7 @@ module snabb-softwire-v1 { } leaf seconds { - type PositiveNumber; + type positive-uint32; default 2; description "The time period given in seconds."; } @@ -156,25 +156,25 @@ module snabb-softwire-v1 { outgoing ICMPv4 and ICMPv6 messages."; leaf icmpv4-incoming { - type PacketPolicy; + type packet-policy; default allow; description "Sets the policy for incoming ICMPv4 messages."; } leaf icmpv4-outgoing { - type PacketPolicy; + type packet-policy; default allow; description "Sets the policy for outgoing ICMPv4 messages."; } leaf icmpv6-incoming { - type PacketPolicy; + type packet-policy; default allow; description "Sets the policy for incoming ICMPv6 messages."; } leaf icmpv6-outgoing { - type PacketPolicy; + type packet-policy; default allow; description "Sets the policy for outgoing ICMPv6 messages."; } @@ -191,12 +191,12 @@ module snabb-softwire-v1 { } leaf v4-tag { - type VlanTag; + type vlan-tag; description "Sets the tag for v4"; } leaf v6-tag { - type VlanTag; + type vlan-tag; description "Sets the tag for v6"; } } From cac171c94e74f3794a9784a532ecc3fda5db3095 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 10:48:14 +0100 Subject: [PATCH 108/631] snabb-softwire-v1: Whitespace/syntax tweaks --- src/lib/yang/snabb-softwire-v1.yang | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index e825e77a52..96c12f84db 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -2,9 +2,9 @@ module snabb-softwire-v1 { namespace snabb:lwaftr; prefix softwire; - import ietf-inet-types {prefix inet; } + import ietf-inet-types { prefix inet; } - organization "Igalia"; + organization "Igalia, S.L."; contact "Jessica Tallon "; description "Configuration for the Snabb Switch lwAFTR."; @@ -19,7 +19,7 @@ module snabb-softwire-v1 { value 0; description "Allow pakcets."; } - enum deny { + enum deny { value 1; description "Deny packets."; } @@ -30,14 +30,14 @@ module snabb-softwire-v1 { typedef positive-uint32 { type uint32 { - range "1.. 2147483647"; + range 1..2147483647; } description "This is a non-negative integer value which is 1 or above."; } typedef vlan-tag { type uint16 { - range "0..4095"; + range 0..4095; } description "This is a VLAN tag according to 802.1Q Ethernet tagging."; } @@ -76,7 +76,6 @@ module snabb-softwire-v1 { RFC 7596."; } - container mtu { description "The MTU settings are used to determine whether a packet needs to be fragmented. The MTU handling is otherwise @@ -130,7 +129,6 @@ module snabb-softwire-v1 { } } - container rate-limiting-icmp { description "ICMP rate limiting is mandated by many RFCs. This configures the number of ICMP packets the lwAFTR can send for the given @@ -289,7 +287,7 @@ module snabb-softwire-v1 { } list br-addresses { - key "ipv6-address"; + key ipv6-address; leaf ipv6-address { type inet:ipv6-address; From 352a02fcb3038fd26e05c58c54d8e150649808f2 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 11:54:30 +0100 Subject: [PATCH 109/631] snabb-softwire: indent and reflow 2-space indents throughout, and follow ietf-inet-types for description indenting. --- src/lib/yang/snabb-softwire-v1.yang | 282 ++++++++++++++++------------ 1 file changed, 163 insertions(+), 119 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 96c12f84db..df1c6679f3 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -6,324 +6,368 @@ module snabb-softwire-v1 { organization "Igalia, S.L."; contact "Jessica Tallon "; - description "Configuration for the Snabb Switch lwAFTR."; + description + "Configuration for the Snabb Switch lwAFTR."; revision 2016-11-04 { - description "Initial revision."; - reference "-00"; + description + "Initial revision."; } typedef packet-policy { type enumeration { enum allow { value 0; - description "Allow pakcets."; + description + "Allow pakcets."; } enum deny { value 1; - description "Deny packets."; + description + "Deny packets."; } } - description "Policy for what to do with packets, either allow or deny - them."; + description + "Policy for what to do with packets, either allow or deny + them."; } typedef positive-uint32 { type uint32 { range 1..2147483647; } - description "This is a non-negative integer value which is 1 or above."; + description + "This is a non-negative integer value which is 1 or above."; } typedef vlan-tag { type uint16 { range 0..4095; } - description "This is a VLAN tag according to 802.1Q Ethernet tagging."; + description + "This is a VLAN tag according to 802.1Q Ethernet tagging."; } container softwire-config { - description "Configuration for Snabb lwaftr."; + description + "Configuration for Snabb lwaftr."; leaf ipv4-ip { type inet:ipv4-address; mandatory true; - description "Sets the IPv4 address of the internet facing NIC."; + description + "Sets the IPv4 address of the internet facing NIC."; } leaf ipv6-ip { type inet:ipv6-address; mandatory true; - description "Sets the IPv6 address of the internal facing NIC."; + description + "Sets the IPv6 address of the internal facing NIC."; } leaf mac-b4 { type inet:mac-address; mandatory true; - description "Sets the MAC address of the internal facing NIC."; + description + "Sets the MAC address of the internal facing NIC."; } leaf mac-inet { type inet:mac-address; mandatory true; - description "Sets the MAC address of the internet facing NIC."; + description + "Sets the MAC address of the internet facing NIC."; } leaf hairpinning { type boolean; default true; - description "Enables or disables hairpinning that is a requirement of - RFC 7596."; + description + "Enables or disables hairpinning that is a requirement of RFC + 7596."; } container mtu { - description "The MTU settings are used to determine whether a packet - needs to be fragmented. The MTU handling is otherwise - underdeveloped. It is not dynamically updated upon receiving - ICMP packets too big."; + description + "The MTU settings are used to determine whether a packet + needs to be fragmented. The MTU handling is otherwise + underdeveloped. It is not dynamically updated upon receiving + ICMP packets too big."; leaf ipv4-mtu { type uint16; default 1460; - description "Sets the MTU value for the IPv4 NIC."; + description + "Sets the MTU value for the IPv4 NIC."; } leaf ipv6-mtu { type uint16; default 1500; - description "Set the MTU value for the IPv6 NIC."; + description + "Set the MTU value for the IPv6 NIC."; } } container L2-next-hop { - description "One might expect to set the default gateway IPv4 and IPv6 - addresses and have those resolved via ARP, however ARP is - not yet supported via the lwAFTR. One must specify either - the MAC Address of the next hop host however you could - instead specify the IP addresses of the next hops and have - the lwAFTR resolve the corresponding MAC addresses via NDP - and ARP."; + description + "One might expect to set the default gateway IPv4 and + IPv6 addresses and have those resolved via ARP, however ARP + is not yet supported via the lwAFTR. One must specify either + the MAC Address of the next hop host however you could + instead specify the IP addresses of the next hops and have + the lwAFTR resolve the corresponding MAC addresses via NDP + and ARP."; leaf b4-mac { type inet:mac-address; - description "Sets the MAC address of the next hop for the internal - facing NIC."; + description + "Sets the MAC address of the next hop for the internal + facing NIC."; } leaf inet-mac { type inet:mac-address; - description "Sets the MAC address of the next hop for the internet - facing NIC."; + description + "Sets the MAC address of the next hop for the internet + facing NIC."; } leaf inet-ipv4 { type inet:ipv4-address; - description "Sets the IPv4 address of the next hop for the internet - facing NIC that will be used to resolve the MAC address."; + description + "Sets the IPv4 address of the next hop for the internet + facing NIC that will be used to resolve the MAC address."; } leaf b4-ipv6 { type inet:ipv6-address; - description "Sets the IPv4 address of the next hop for the internal - facing NIC that will be used to resolve the MAC address."; + description + "Sets the IPv4 address of the next hop for the internal + facing NIC that will be used to resolve the MAC address."; } } container rate-limiting-icmp { - description "ICMP rate limiting is mandated by many RFCs. This configures - the number of ICMP packets the lwAFTR can send for the given - time period. Lower values are recomended for - non-experimental use."; + description + "ICMP rate limiting is mandated by many RFCs. This configures + the number of ICMP packets the lwAFTR can send for the given + time period. Lower values are recomended for + non-experimental use."; leaf packets { type uint32; default 600000; - description "The number of packets which can be sent within the time - period."; + description + "The number of packets which can be sent within the time + period."; } leaf seconds { type positive-uint32; default 2; - description "The time period given in seconds."; + description + "The time period given in seconds."; } } container icmp-handling-policies { - description "The lwAFTR can be configured to allow or drop incoming and - outgoing ICMPv4 and ICMPv6 messages."; + description + "The lwAFTR can be configured to allow or drop incoming and + outgoing ICMPv4 and ICMPv6 messages."; leaf icmpv4-incoming { type packet-policy; default allow; - description "Sets the policy for incoming ICMPv4 messages."; + description + "Sets the policy for incoming ICMPv4 messages."; } leaf icmpv4-outgoing { type packet-policy; default allow; - description "Sets the policy for outgoing ICMPv4 messages."; + description + "Sets the policy for outgoing ICMPv4 messages."; } leaf icmpv6-incoming { type packet-policy; default allow; - description "Sets the policy for incoming ICMPv6 messages."; + description + "Sets the policy for incoming ICMPv6 messages."; } leaf icmpv6-outgoing { type packet-policy; default allow; - description "Sets the policy for outgoing ICMPv6 messages."; + description + "Sets the policy for outgoing ICMPv6 messages."; } } container vlan-tagging { - description "This configures 802.1Q Ethernet tagging. If tagging is - enabled the outbound and inbound tags must be specified."; + description + "This configures 802.1Q Ethernet tagging. If tagging is + enabled the outbound and inbound tags must be specified."; leaf vlan-tagging { type boolean; default true; - description "Enables or disables VLAN tagging."; + description + "Enables or disables VLAN tagging."; } leaf v4-tag { type vlan-tag; - description "Sets the tag for v4"; + description + "Sets the tag for v4"; } leaf v6-tag { type vlan-tag; - description "Sets the tag for v6"; + description + "Sets the tag for v6"; } } container filters { - description "This configures the ingress and egress filters for each NIC. - If set, these should be a pflang filter, pflang is the - language of tcpdump, libpcap and other tools. The filters - will run on the packets without vlan tags, if any."; + description + "This configures the ingress and egress filters for each + NIC. If set, these should be a pflang filter, pflang is the + language of tcpdump, libpcap and other tools. The filters + will run on the packets without vlan tags, if any."; leaf ipv4-ingress-filter { type string; - description "Sets the pflang filter for IPv4 ingress."; + description + "Sets the pflang filter for IPv4 ingress."; } leaf ipv4-egress-filter { type string; - description "Sets the pflang filter for IPv4 egress."; + description + "Sets the pflang filter for IPv4 egress."; } leaf ipv6-ingress-filter { type string; - description "Sets the pflang filter for IPv6 ingress."; + description + "Sets the pflang filter for IPv6 ingress."; } leaf ipv6-egress-filter { type string; - description "Sets the pflang filter for IPv6 egress."; + description + "Sets the pflang filter for IPv6 egress."; } } } container binding-table { - description "The binding table is a collection of softwire tunnels. One - endpoint of the softwire is the lwAFTR and the other is the - B4. It contains information so the lwAFTR knows which IPv4 - (or part of an IPv4 address) is associated with which B4."; + description + "The binding table is a collection of softwire tunnels. One + endpoint of the softwire is the lwAFTR and the other is the + B4. It contains information so the lwAFTR knows which IPv4 (or + part of an IPv4 address) is associated with which B4."; list binding-entry { key "binding-ipv6info"; - description "Defines the set of IPv4 addresses that are provisioned by a - lwAFTR. It also defines the way in which those addresses are - shared by specifying the psid_length and shift. See RFC 7597 - for more details on the PSID scheme for sharing IPv4 - addresses."; + description + "Defines the set of IPv4 addresses that are provisioned by a + lwAFTR. It also defines the way in which those addresses are + shared by specifying the psid_length and shift. See RFC 7597 + for more details on the PSID scheme for sharing IPv4 + addresses."; leaf binding-ipv6info { - type union { - type inet:ipv6-address; - type inet:ipv6-prefix; - } - mandatory false; + type union { + type inet:ipv6-address; + type inet:ipv6-prefix; + } + mandatory false; } leaf binding-ipv4-addr { - type inet:ipv4-address; - mandatory true; + type inet:ipv4-address; + mandatory true; } container port-set { - leaf psid-offset { - type uint8 { - range 0..16; - } - mandatory true; - } - - leaf psid-len { - type uint8 { - range 0..15; - } - mandatory true; - } - - leaf psid { - type uint16; - mandatory true; - } + leaf psid-offset { + type uint8 { + range 0..16; + } + mandatory true; + } + + leaf psid-len { + type uint8 { + range 0..15; + } + mandatory true; + } + + leaf psid { + type uint16; + mandatory true; + } } leaf br-ipv6-addr { - type inet:ipv6-address; - mandatory true; + type inet:ipv6-address; + mandatory true; } leaf lifetime { - type uint32; - units seconds; + type uint32; + units seconds; } - } + } - list br-addresses { + list br-addresses { key ipv6-address; leaf ipv6-address { type inet:ipv6-address; } - } + } - list binding-table-entry { + list binding-table-entry { key "ipv4-address psid"; leaf ipv4-address { - type inet:ipv4-address; - mandatory true; - description "Public IPv4 address of the softwire."; + type inet:ipv4-address; + mandatory true; + description + "Public IPv4 address of the softwire."; } leaf psid { - type uint16; - mandatory true; - description "Port set ID"; + type uint16; + mandatory true; + description + "Port set ID"; } leaf padding { - type uint16; - description "Zeros"; + type uint16; + description + "Zeros"; } leaf br { - type uint32; - mandatory true; - description "Border router"; + type uint32; + mandatory true; + description + "Border router"; } leaf ipv6-address { - type inet:ipv6-address; - mandatory true; - description "Address of the B4"; + type inet:ipv6-address; + mandatory true; + description + "Address of the B4"; } } } From 36c866f1a0079d2e086936240cb3ad6771ee7000 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 11:56:46 +0100 Subject: [PATCH 110/631] snabb-softwire: Two spaces after period in descriptions --- src/lib/yang/snabb-softwire-v1.yang | 42 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index df1c6679f3..ff812dbe9a 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -19,7 +19,7 @@ module snabb-softwire-v1 { enum allow { value 0; description - "Allow pakcets."; + "Allow packets."; } enum deny { value 1; @@ -91,9 +91,9 @@ module snabb-softwire-v1 { container mtu { description "The MTU settings are used to determine whether a packet - needs to be fragmented. The MTU handling is otherwise - underdeveloped. It is not dynamically updated upon receiving - ICMP packets too big."; + needs to be fragmented. The MTU handling is otherwise + underdeveloped. It is not dynamically updated upon + receiving ICMP packets too big."; leaf ipv4-mtu { type uint16; @@ -114,11 +114,11 @@ module snabb-softwire-v1 { description "One might expect to set the default gateway IPv4 and IPv6 addresses and have those resolved via ARP, however ARP - is not yet supported via the lwAFTR. One must specify either - the MAC Address of the next hop host however you could - instead specify the IP addresses of the next hops and have - the lwAFTR resolve the corresponding MAC addresses via NDP - and ARP."; + is not yet supported via the lwAFTR. One must specify + either the MAC Address of the next hop host however you + could instead specify the IP addresses of the next hops and + have the lwAFTR resolve the corresponding MAC addresses via + NDP and ARP."; leaf b4-mac { type inet:mac-address; @@ -151,9 +151,9 @@ module snabb-softwire-v1 { container rate-limiting-icmp { description - "ICMP rate limiting is mandated by many RFCs. This configures - the number of ICMP packets the lwAFTR can send for the given - time period. Lower values are recomended for + "ICMP rate limiting is mandated by many RFCs. This + configures the number of ICMP packets the lwAFTR can send + for the given time period. Lower values are recomended for non-experimental use."; leaf packets { @@ -208,7 +208,7 @@ module snabb-softwire-v1 { container vlan-tagging { description - "This configures 802.1Q Ethernet tagging. If tagging is + "This configures 802.1Q Ethernet tagging. If tagging is enabled the outbound and inbound tags must be specified."; leaf vlan-tagging { @@ -234,8 +234,8 @@ module snabb-softwire-v1 { container filters { description "This configures the ingress and egress filters for each - NIC. If set, these should be a pflang filter, pflang is the - language of tcpdump, libpcap and other tools. The filters + NIC. If set, these should be a pflang filter, pflang is the + language of tcpdump, libpcap and other tools. The filters will run on the packets without vlan tags, if any."; leaf ipv4-ingress-filter { @@ -266,18 +266,18 @@ module snabb-softwire-v1 { container binding-table { description - "The binding table is a collection of softwire tunnels. One + "The binding table is a collection of softwire tunnels. One endpoint of the softwire is the lwAFTR and the other is the - B4. It contains information so the lwAFTR knows which IPv4 (or - part of an IPv4 address) is associated with which B4."; + B4. It contains information so the lwAFTR knows which IPv4 + (or part of an IPv4 address) is associated with which B4."; list binding-entry { key "binding-ipv6info"; description "Defines the set of IPv4 addresses that are provisioned by a - lwAFTR. It also defines the way in which those addresses are - shared by specifying the psid_length and shift. See RFC 7597 - for more details on the PSID scheme for sharing IPv4 + lwAFTR. It also defines the way in which those addresses + are shared by specifying the psid_length and shift. See RFC + 7597 for more details on the PSID scheme for sharing IPv4 addresses."; leaf binding-ipv6info { From 9ffe80daf7b82d98cdc31dee9f5872035dd0690d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 11:58:44 +0100 Subject: [PATCH 111/631] snabb-softwire: typedefs closer to uses --- src/lib/yang/snabb-softwire-v1.yang | 62 +++++++++++++---------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index ff812dbe9a..81b0d7803d 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -14,40 +14,6 @@ module snabb-softwire-v1 { "Initial revision."; } - typedef packet-policy { - type enumeration { - enum allow { - value 0; - description - "Allow packets."; - } - enum deny { - value 1; - description - "Deny packets."; - } - } - description - "Policy for what to do with packets, either allow or deny - them."; - } - - typedef positive-uint32 { - type uint32 { - range 1..2147483647; - } - description - "This is a non-negative integer value which is 1 or above."; - } - - typedef vlan-tag { - type uint16 { - range 0..4095; - } - description - "This is a VLAN tag according to 802.1Q Ethernet tagging."; - } - container softwire-config { description "Configuration for Snabb lwaftr."; @@ -165,7 +131,7 @@ module snabb-softwire-v1 { } leaf seconds { - type positive-uint32; + type uint32 { range 1..max } default 2; description "The time period given in seconds."; @@ -177,6 +143,24 @@ module snabb-softwire-v1 { "The lwAFTR can be configured to allow or drop incoming and outgoing ICMPv4 and ICMPv6 messages."; + typedef packet-policy { + type enumeration { + enum allow { + value 0; + description + "Allow packets."; + } + enum deny { + value 1; + description + "Deny packets."; + } + } + description + "Policy for what to do with packets, either allow or deny + them."; + } + leaf icmpv4-incoming { type packet-policy; default allow; @@ -211,6 +195,14 @@ module snabb-softwire-v1 { "This configures 802.1Q Ethernet tagging. If tagging is enabled the outbound and inbound tags must be specified."; + typedef vlan-tag { + type uint16 { + range 0..4095; + } + description + "This is a VLAN tag according to 802.1Q Ethernet tagging."; + } + leaf vlan-tagging { type boolean; default true; From be63713910694de44736169fe79fba0f8f1b868d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 12:24:22 +0100 Subject: [PATCH 112/631] snabb-softwire: descriptions more declarative --- src/lib/yang/snabb-softwire-v1.yang | 92 ++++++++++++++--------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 81b0d7803d..a9e18f7f23 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -22,112 +22,111 @@ module snabb-softwire-v1 { type inet:ipv4-address; mandatory true; description - "Sets the IPv4 address of the internet facing NIC."; + "IPv4 address of the internet facing NIC."; } leaf ipv6-ip { type inet:ipv6-address; mandatory true; description - "Sets the IPv6 address of the internal facing NIC."; + "IPv6 address of the internal facing NIC."; } leaf mac-b4 { type inet:mac-address; mandatory true; description - "Sets the MAC address of the internal facing NIC."; + "MAC address of the internal facing NIC."; } leaf mac-inet { type inet:mac-address; mandatory true; description - "Sets the MAC address of the internet facing NIC."; + "MAC address of the internet facing NIC."; } leaf hairpinning { type boolean; default true; description - "Enables or disables hairpinning that is a requirement of RFC - 7596."; + "Indicates whether to support hairpinning of traffic between + two B4s."; } container mtu { description "The MTU settings are used to determine whether a packet - needs to be fragmented. The MTU handling is otherwise - underdeveloped. It is not dynamically updated upon - receiving ICMP packets too big."; + needs to be fragmented. Note that the MTU settings are + static; they do not update if a peer sends an ICMP + packet-too-big message."; leaf ipv4-mtu { type uint16; default 1460; description - "Sets the MTU value for the IPv4 NIC."; + "Maximum packet size to send on the IPv4 interface."; } leaf ipv6-mtu { type uint16; default 1500; description - "Set the MTU value for the IPv6 NIC."; + "Maximum packet size to sent on the IPv6 interface."; } } container L2-next-hop { description - "One might expect to set the default gateway IPv4 and - IPv6 addresses and have those resolved via ARP, however ARP - is not yet supported via the lwAFTR. One must specify - either the MAC Address of the next hop host however you - could instead specify the IP addresses of the next hops and - have the lwAFTR resolve the corresponding MAC addresses via - NDP and ARP."; + "The next-hop settings are used to indicate the next hops on + the internet-facing and internal-facing interfaces. + + One can specify a next-hop as either layer 3 or a layer 2 + address. In the former case, the lwAFTR will use ARP or NDP + (as appropriate) to resolve the L2 addresses. In the latter + case, the L2 next-hop addresses instead are statically + configured by the administrator." leaf b4-mac { type inet:mac-address; description - "Sets the MAC address of the next hop for the internal - facing NIC."; + "Statically configured MAC address of the next hop for the + internal-facing NIC."; } leaf inet-mac { type inet:mac-address; description - "Sets the MAC address of the next hop for the internet - facing NIC."; + "Statically configured MAC address of the next hop for the + internet-facing NIC."; } leaf inet-ipv4 { type inet:ipv4-address; description - "Sets the IPv4 address of the next hop for the internet - facing NIC that will be used to resolve the MAC address."; + "IPv4 address of the next hop for the internet-facing NIC. + The lwAFTR will resolve this to a MAC address using ARP."; } leaf b4-ipv6 { type inet:ipv6-address; description - "Sets the IPv4 address of the next hop for the internal - facing NIC that will be used to resolve the MAC address."; + "IPv6 address of the next hop for the internal-facing NIC. + The lwAFTR will resolve this to a MAC address using NDP."; } } container rate-limiting-icmp { description - "ICMP rate limiting is mandated by many RFCs. This - configures the number of ICMP packets the lwAFTR can send - for the given time period. Lower values are recomended for - non-experimental use."; + "These settings limit the rate of ICMP error message + transmission."; leaf packets { type uint32; default 600000; description - "The number of packets which can be sent within the time - period."; + "The number of ICMP error messages which can be sent within + the time period."; } leaf seconds { @@ -165,28 +164,28 @@ module snabb-softwire-v1 { type packet-policy; default allow; description - "Sets the policy for incoming ICMPv4 messages."; + "Whether to allow incoming ICMPv4 messages."; } leaf icmpv4-outgoing { type packet-policy; default allow; description - "Sets the policy for outgoing ICMPv4 messages."; + "Whether to generate outgoing ICMPv4 error amessages."; } leaf icmpv6-incoming { type packet-policy; default allow; description - "Sets the policy for incoming ICMPv6 messages."; + "Whether to allow incoming ICMPv6 messages."; } leaf icmpv6-outgoing { type packet-policy; default allow; description - "Sets the policy for outgoing ICMPv6 messages."; + "Whether to generate outgoing ICMPv6 error messages."; } } @@ -200,7 +199,7 @@ module snabb-softwire-v1 { range 0..4095; } description - "This is a VLAN tag according to 802.1Q Ethernet tagging."; + "An 802.1Q Ethernet VLAN tag."; } leaf vlan-tagging { @@ -213,45 +212,46 @@ module snabb-softwire-v1 { leaf v4-tag { type vlan-tag; description - "Sets the tag for v4"; + "VLAN tag for IPv4 interface."; } leaf v6-tag { type vlan-tag; description - "Sets the tag for v6"; + "VLAN tag for IPv6 interface."; } } container filters { description "This configures the ingress and egress filters for each - NIC. If set, these should be a pflang filter, pflang is the - language of tcpdump, libpcap and other tools. The filters - will run on the packets without vlan tags, if any."; + NIC. If set, these should be a pflang filter. pflang is + the language of tcpdump, libpcap and other tools. Note that + if VLAN tagging is enabled, the the filters will run on the + packets after VLAN tags have been stripped off."; leaf ipv4-ingress-filter { type string; description - "Sets the pflang filter for IPv4 ingress."; + "IPv4 ingress filter."; } leaf ipv4-egress-filter { type string; description - "Sets the pflang filter for IPv4 egress."; + "IPv4 egress filter."; } leaf ipv6-ingress-filter { type string; description - "Sets the pflang filter for IPv6 ingress."; + "IPv6 ingress filter."; } leaf ipv6-egress-filter { type string; description - "Sets the pflang filter for IPv6 egress."; + "IPv6 egress filter."; } } } From 9e4be097ed90fefa96fd19ba9c0629139265abc6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 12:29:43 +0100 Subject: [PATCH 113/631] snabb-softwire: Update binding table This patch updates the binding table part of snabb-softwire to match the model used by the data plane. --- src/lib/yang/snabb-softwire-v1.yang | 148 +++++++++++++++++----------- 1 file changed, 89 insertions(+), 59 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index a9e18f7f23..9389c4490b 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -258,75 +258,103 @@ module snabb-softwire-v1 { container binding-table { description - "The binding table is a collection of softwire tunnels. One - endpoint of the softwire is the lwAFTR and the other is the - B4. It contains information so the lwAFTR knows which IPv4 - (or part of an IPv4 address) is associated with which B4."; + "A collection of softwires (tunnels), along with a description + of the IPv4 and IPv6 addresses handled by the lwAFTR."; - list binding-entry { - key "binding-ipv6info"; - description - "Defines the set of IPv4 addresses that are provisioned by a - lwAFTR. It also defines the way in which those addresses - are shared by specifying the psid_length and shift. See RFC - 7597 for more details on the PSID scheme for sharing IPv4 - addresses."; - - leaf binding-ipv6info { - type union { - type inet:ipv6-address; - type inet:ipv6-prefix; - } - mandatory false; - } + presence true; - leaf binding-ipv4-addr { - type inet:ipv4-address; + list psid-map { + description + "The set of IPv4 addresses managed by the lwAFTR, along with + the way in which those IPv4 addresses share ports. A PSID map + entry associates a PSID length, shift, and + reserved-ports-bit-count with each IPv4 address served by + the lwAFTR. + + The lightweight 4-over-6 architecture supports sharing of + IPv4 addresses by partitioning the space of TCP/UDP/ICMP + ports into disjoint \"port sets\". Each softwire associated + with an IPv4 address corresponds to a different set of ports + on that address. The way that the ports are partitioned is + specified in RFC 7597: each address has an associated set + of parameters that specifies how to compute a \"port set + identifier\" (PSID) from a given port. + + 0 1 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + +-----------+-----------+-------+ + Ports in | A | PSID | j | + the CE port set | > 0 | | | + +-----------+-----------+-------+ + | a bits | k bits |m bits | + + Figure 2: Structure of a Port-Restricted Port Field + + Source: http://tools.ietf.org/html/rfc7597#section-5.1 + + We find the specification's names to be a bit obtuse, so we + refer to them using the following names: + + a bits = reserved-ports-bit-count. + k bits = psid-length. + m bits = shift."; + + key addr; + + leaf addr { + type ipv4-address; mandatory true; + description + "Public IPv4 address managed by the lwAFTR." } - container port-set { - leaf psid-offset { - type uint8 { - range 0..16; - } - mandatory true; - } - - leaf psid-len { - type uint8 { - range 0..15; - } - mandatory true; - } - - leaf psid { - type uint16; - mandatory true; - } + leaf end-addr { + type ipv4-address; + description + "If present, this PSID map entry applies to all addresses + between 'addr' and this address, inclusive." } - leaf br-ipv6-addr { - type inet:ipv6-address; + leaf psid-length { + type uint8 { range 0..16; } mandatory true; + description + "The number of bits devoted to the PSID in the port map. + If the psid-length is N, then the IPv4 address will be + shared 2^N ways. Note that psid-length, shift, and + reserved-ports-bit-count must add up to 16." } - leaf lifetime { - type uint32; - units seconds; + leaf shift { + type uint8 { range 0..16; } + description + "Given an incoming port, one can obtain the PSID by + shifting the port right by 'shift' bits and then masking + off the lowest 'psid-length' bits. Defaults to 16 - + psid-length. Note that psid-length, shift, and + reserved-ports-bit-count must add up to 16." } - } - - list br-addresses { - key ipv6-address; - leaf ipv6-address { - type inet:ipv6-address; + leaf reserved-ports-bit-count { + type uint8 { range 0..16; } + default 0; + description + "Reserve the lowest 2^N ports so that they map to no + softwire. This can be useful to prevent the low 1024 + ports (for example) from being mapped to customers. Note + that psid-length and shift must add up to less than or + equal to 16." } } - list binding-table-entry { - key "ipv4-address psid"; + leaf-list br-address { + type ipv6-address; + description + "B4-facing address of an lwAFTR." + } + + list softwire { + key "ipv4-address psid padding"; leaf ipv4-address { type inet:ipv4-address; @@ -339,27 +367,29 @@ module snabb-softwire-v1 { type uint16; mandatory true; description - "Port set ID"; + "Port set ID."; } leaf padding { - type uint16; + type uint16 { range 0..0 }; + default 0; description - "Zeros"; + "Reserved bytes."; } leaf br { type uint32; mandatory true; description - "Border router"; + "The B4-facing address of the lwAFTR for this softwire, as + a zero-based index into br-addresses."; } leaf ipv6-address { type inet:ipv6-address; mandatory true; description - "Address of the B4"; + "B4 address."; } } } From a6d6723ec650484799ca4e15bde80887a068d101 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 13:31:42 +0100 Subject: [PATCH 114/631] snabb-softwire: Reorg into "external" and "internal" --- src/lib/yang/snabb-softwire-v1.yang | 305 ++++++++++++---------------- 1 file changed, 126 insertions(+), 179 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 9389c4490b..55909f21f2 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -3,6 +3,7 @@ module snabb-softwire-v1 { prefix softwire; import ietf-inet-types { prefix inet; } + import ietf-yang-types { prefix yang; } organization "Igalia, S.L."; contact "Jessica Tallon "; @@ -18,240 +19,186 @@ module snabb-softwire-v1 { description "Configuration for Snabb lwaftr."; - leaf ipv4-ip { - type inet:ipv4-address; - mandatory true; + grouping traffic-filters { + type string; description - "IPv4 address of the internet facing NIC."; - } - - leaf ipv6-ip { - type inet:ipv6-address; - mandatory true; - description - "IPv6 address of the internal facing NIC."; - } - - leaf mac-b4 { - type inet:mac-address; - mandatory true; - description - "MAC address of the internal facing NIC."; - } - - leaf mac-inet { - type inet:mac-address; - mandatory true; - description - "MAC address of the internet facing NIC."; - } - - leaf hairpinning { - type boolean; - default true; - description - "Indicates whether to support hairpinning of traffic between - two B4s."; + "Ingress and egress filters describing the set of packets + that should be allowed to pass, as pflang filters. pflang + is the language of tcpdump, libpcap and other tools. Note + that if VLAN tagging is enabled, the filters run on packets + after VLAN tags have been stripped off."; + leaf ingress-filter { + type string; + description + "Filter for incoming traffic. Packets that do not match + the filter will be silently dropped."; + } + leaf egress-filter { + type string; + description + "Filter for outgoing traffic. Packets that do not match + the filter will be silently dropped."; + } } - container mtu { + grouping icmp-policy { description - "The MTU settings are used to determine whether a packet - needs to be fragmented. Note that the MTU settings are - static; they do not update if a peer sends an ICMP - packet-too-big message."; + "The lwAFTR can be configured to allow or drop incoming ICMP + messages, and to generate outgoing ICMP error messages or + not."; - leaf ipv4-mtu { - type uint16; - default 1460; + leaf allow-incoming-icmp { + type boolean; + default true; description - "Maximum packet size to send on the IPv4 interface."; + "Whether to allow incoming ICMP packets."; } - leaf ipv6-mtu { - type uint16; - default 1500; + leaf generate-icmp-errors { + type boolean; + default true; description - "Maximum packet size to sent on the IPv6 interface."; + "Whether to generate outgoing ICMP error messages."; } } - container L2-next-hop { + grouping vlan-tagging { description - "The next-hop settings are used to indicate the next hops on - the internet-facing and internal-facing interfaces. - - One can specify a next-hop as either layer 3 or a layer 2 - address. In the former case, the lwAFTR will use ARP or NDP - (as appropriate) to resolve the L2 addresses. In the latter - case, the L2 next-hop addresses instead are statically - configured by the administrator." + "802.1Q Ethernet tagging."; - leaf b4-mac { - type inet:mac-address; + leaf vlan-tag { + type uint16 { + range 0..4095; + } description - "Statically configured MAC address of the next hop for the - internal-facing NIC."; + "802.1Q Ethernet VLAN tag for this interface."; } + } - leaf inet-mac { - type inet:mac-address; - description - "Statically configured MAC address of the next hop for the - internet-facing NIC."; - } + container external-interface { + description + "Configuration for the external, internet-facing IPv4 + interface."; + + presence true; - leaf inet-ipv4 { + leaf ip { type inet:ipv4-address; + mandatory true; description - "IPv4 address of the next hop for the internet-facing NIC. - The lwAFTR will resolve this to a MAC address using ARP."; + "L3 Address of the internet-facing network interface. Used + when generating error messages and responding to ICMP echo + requests."; } - - leaf b4-ipv6 { - type inet:ipv6-address; + leaf mac { + type yang:mac-address; + mandatory true; description - "IPv6 address of the next hop for the internal-facing NIC. - The lwAFTR will resolve this to a MAC address using NDP."; + "MAC address of the internet-facing NIC."; } - } - - container rate-limiting-icmp { - description - "These settings limit the rate of ICMP error message - transmission."; - - leaf packets { - type uint32; - default 600000; + leaf mtu { + type uint16; + default 1460; description - "The number of ICMP error messages which can be sent within - the time period."; + "Maximum packet size to send on the IPv4 interface."; } - leaf seconds { - type uint32 { range 1..max } - default 2; - description - "The time period given in seconds."; + using traffic-filters; + using icmp-policy; + using vlan-tagging; + + container next-hop { + presence true; + leaf ip { + type inet:ipv4-address; + description + "IPv4 address of the next hop for the internet-facing NIC. + The lwAFTR will resolve this to a MAC address using ARP."; + } + leaf mac { + type yang:mac-address; + description + "Statically configured MAC address of the next hop for the + internet-facing NIC."; + } } } - container icmp-handling-policies { + container internal-interface { description - "The lwAFTR can be configured to allow or drop incoming and - outgoing ICMPv4 and ICMPv6 messages."; - - typedef packet-policy { - type enumeration { - enum allow { - value 0; - description - "Allow packets."; - } - enum deny { - value 1; - description - "Deny packets."; - } - } - description - "Policy for what to do with packets, either allow or deny - them."; - } + "Configuration for the internal IPv6 interface."; - leaf icmpv4-incoming { - type packet-policy; - default allow; - description - "Whether to allow incoming ICMPv4 messages."; - } + presence true; - leaf icmpv4-outgoing { - type packet-policy; - default allow; + leaf ip { + type inet:ipv6-address; + mandatory true; description - "Whether to generate outgoing ICMPv4 error amessages."; + "L3 Address of the internal-facing network interface. Used + when generating error messages and responding to ICMP echo + requests."; } - - leaf icmpv6-incoming { - type packet-policy; - default allow; + leaf mac { + type inet:mac-address; + mandatory true; description - "Whether to allow incoming ICMPv6 messages."; + "MAC address of the internal-facing NIC."; } - - leaf icmpv6-outgoing { - type packet-policy; - default allow; + leaf mtu { + type uint16; + default 1500; description - "Whether to generate outgoing ICMPv6 error messages."; + "Maximum packet size to sent on the IPv6 interface."; } - } - container vlan-tagging { - description - "This configures 802.1Q Ethernet tagging. If tagging is - enabled the outbound and inbound tags must be specified."; - - typedef vlan-tag { - type uint16 { - range 0..4095; + using traffic-filters; + using icmp-policy; + using vlan-tagging; + + container next-hop { + presence true; + leaf ip { + type inet:ipv6-address; + description + "IPv6 address of the next hop for the internal-facing NIC. + The lwAFTR will resolve this to a MAC address using NDP."; + } + leaf mac { + type inet:mac-address; + description + "Statically configured MAC address of the next hop for the + internal-facing NIC."; } - description - "An 802.1Q Ethernet VLAN tag."; } - leaf vlan-tagging { + leaf hairpinning { type boolean; default true; description - "Enables or disables VLAN tagging."; - } - - leaf v4-tag { - type vlan-tag; - description - "VLAN tag for IPv4 interface."; - } - - leaf v6-tag { - type vlan-tag; - description - "VLAN tag for IPv6 interface."; + "Indicates whether to support hairpinning of traffic between + two B4s."; } } - container filters { + container error-rate-limiting { description - "This configures the ingress and egress filters for each - NIC. If set, these should be a pflang filter. pflang is - the language of tcpdump, libpcap and other tools. Note that - if VLAN tagging is enabled, the the filters will run on the - packets after VLAN tags have been stripped off."; - - leaf ipv4-ingress-filter { - type string; - description - "IPv4 ingress filter."; - } + "These settings limit the rate of ICMP error message + transmission."; - leaf ipv4-egress-filter { - type string; - description - "IPv4 egress filter."; - } + presence true; - leaf ipv6-ingress-filter { - type string; + leaf packets { + type uint32; description - "IPv6 ingress filter."; + "The number of ICMP error messages which can be sent within + the specified time period."; } - leaf ipv6-egress-filter { - type string; + leaf period { + type uint32 { range 1..max } + default 2; description - "IPv6 egress filter."; + "The time period given in seconds."; } } } From a76629993edf82e2f2421cc9d548ad85661e6cab Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 4 Nov 2016 13:38:04 +0100 Subject: [PATCH 115/631] Fix snabb-softwire-v1 syntax errors Also add it to schema selftest. --- src/lib/yang/schema.lua | 3 +++ src/lib/yang/snabb-softwire-v1.yang | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 651d62aaa3..9626af1f1f 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -813,6 +813,7 @@ function load_schema_by_name(name, revision) end function selftest() + print('selftest: lib.yang.schema') local test_schema = [[module fruit { namespace "urn:testing:fruit"; prefix "fruit"; @@ -926,4 +927,6 @@ function selftest() load_schema_by_name('ietf-yang-types') load_schema_by_name('ietf-softwire') + load_schema_by_name('snabb-softwire-v1') + print('selftest: ok') end diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 55909f21f2..1047a2d7f9 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -139,7 +139,7 @@ module snabb-softwire-v1 { requests."; } leaf mac { - type inet:mac-address; + type yang:mac-address; mandatory true; description "MAC address of the internal-facing NIC."; @@ -164,7 +164,7 @@ module snabb-softwire-v1 { The lwAFTR will resolve this to a MAC address using NDP."; } leaf mac { - type inet:mac-address; + type yang:mac-address; description "Statically configured MAC address of the next hop for the internal-facing NIC."; @@ -195,7 +195,7 @@ module snabb-softwire-v1 { } leaf period { - type uint32 { range 1..max } + type uint32 { range 1..max; } default 2; description "The time period given in seconds."; @@ -249,17 +249,17 @@ module snabb-softwire-v1 { key addr; leaf addr { - type ipv4-address; + type inet:ipv4-address; mandatory true; description - "Public IPv4 address managed by the lwAFTR." + "Public IPv4 address managed by the lwAFTR."; } leaf end-addr { - type ipv4-address; + type inet:ipv4-address; description "If present, this PSID map entry applies to all addresses - between 'addr' and this address, inclusive." + between 'addr' and this address, inclusive."; } leaf psid-length { @@ -269,7 +269,7 @@ module snabb-softwire-v1 { "The number of bits devoted to the PSID in the port map. If the psid-length is N, then the IPv4 address will be shared 2^N ways. Note that psid-length, shift, and - reserved-ports-bit-count must add up to 16." + reserved-ports-bit-count must add up to 16."; } leaf shift { @@ -279,7 +279,7 @@ module snabb-softwire-v1 { shifting the port right by 'shift' bits and then masking off the lowest 'psid-length' bits. Defaults to 16 - psid-length. Note that psid-length, shift, and - reserved-ports-bit-count must add up to 16." + reserved-ports-bit-count must add up to 16."; } leaf reserved-ports-bit-count { @@ -290,14 +290,14 @@ module snabb-softwire-v1 { softwire. This can be useful to prevent the low 1024 ports (for example) from being mapped to customers. Note that psid-length and shift must add up to less than or - equal to 16." + equal to 16."; } } leaf-list br-address { - type ipv6-address; + type inet:ipv6-address; description - "B4-facing address of an lwAFTR." + "B4-facing address of an lwAFTR."; } list softwire { @@ -318,7 +318,7 @@ module snabb-softwire-v1 { } leaf padding { - type uint16 { range 0..0 }; + type uint16 { range 0..0; } default 0; description "Reserved bytes."; From 064393a414fb69c43d35083e010f491142132b54 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 4 Nov 2016 18:28:16 +0100 Subject: [PATCH 116/631] Wrongly wrote outgoing v4 packets to v6 link --- src/program/snabbvmx/lwaftr/setup.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index bc7cdc6c9d..21acf920cc 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -376,7 +376,7 @@ local function lwaftr_app_check (c, conf, lwconf, sources, sinks) config.app(c, "nh_fwd4", nh_fwd.nh_fwd4, subset(nh_fwd.nh_fwd4.config, conf.ipv4_interface)) config.link(c, v4_input.."-> nh_fwd4.wire") - config.link(c, "nh_fwd4.wire -> "..v6_output) + config.link(c, "nh_fwd4.wire -> "..v4_output) lwconf.counters = lwcounter.init_counters() config.app(c, "lwaftr", lwaftr.LwAftr, lwconf) From f862b77483c3a8455c821e9ad4e1a2ea1845862f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 4 Nov 2016 19:40:18 +0100 Subject: [PATCH 117/631] Increase SnabbVMX nexthop selftest qemu image RAM to 1Gb --- src/program/snabbvmx/tests/nexthop/selftest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index 36829b35f1..3bc39fa7ed 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -30,6 +30,7 @@ SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr.cfg SNABBVMX_ID=xe1 SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock +GUEST_MEM=1024 function last_32bit { mac=$1 From caa03b8b91f9f038ed50b6ccc61bfec0abb68b78 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 4 Nov 2016 21:22:09 +0100 Subject: [PATCH 118/631] Check only the VM returned MAC addresses different than nil or zero Also increases TIMEOUT to 20 seconds. --- src/program/snabbvmx/tests/nexthop/selftest.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index 3bc39fa7ed..f05756fecd 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -32,11 +32,6 @@ SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock GUEST_MEM=1024 -function last_32bit { - mac=$1 - echo `echo $mac | egrep -o "[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+$"` -} - function cleanup { exit $1 } @@ -56,23 +51,26 @@ if ! snabb $SNABB_PCI1 "packetblaster replay -D 10 $PCAP_INPUT/v4v6-256.pcap $SN fi # Query nexthop for 10 seconds. -TIMEOUT=10 +TIMEOUT=20 count=0 while true; do output=`./snabb snabbvmx nexthop | egrep -o "[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+"` mac_v4=`echo "$output" | head -1` mac_v6=`echo "$output" | tail -1` - # Somehow the first 16-bit of nexhop come from the VM corrupted, compare only last 32-bit. - if [[ $(last_32bit "$mac_v4") == "99:99:99:99" && - $(last_32bit "$mac_v6") == "99:99:99:99" ]]; then + # FIXME: Should return expected MAC addresses. + # Check VM returned something. + if [[ "$mac_v4" != "00:00:00:00:00:00" && + "$mac_v6" != "00:00:00:00:00:00" ]]; then echo "Resolved MAC inet side: $mac_v4 [OK]" - echo "Resolved MAC inet side: $mac_v6 [OK]" + echo "Resolved MAC b4 side: $mac_v6 [OK]" exit 0 fi if [[ $count == $TIMEOUT ]]; then echo "Could not resolve nexthop" + echo "MAC inet side: $mac_v4 [FAILED]" + echo "MAC b4 side: $mac_v6 [FAILED]" exit 1 fi From a0d3ce4908a52310b4eb067e1a9630c18b67c5e5 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 4 Nov 2016 19:40:18 +0100 Subject: [PATCH 119/631] Increase SnabbVMX nexthop selftest qemu image RAM to 1Gb --- src/program/snabbvmx/tests/nexthop/selftest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index 36829b35f1..3bc39fa7ed 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -30,6 +30,7 @@ SNABBVMX_CONF=$SNABBVMX_DIR/tests/conf/snabbvmx-lwaftr.cfg SNABBVMX_ID=xe1 SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock +GUEST_MEM=1024 function last_32bit { mac=$1 From 0ea9f8ad3f30a5d499f3f86bf9c239e15659e20b Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 4 Nov 2016 21:22:09 +0100 Subject: [PATCH 120/631] Check only the VM returned MAC addresses different than nil or zero Also increases TIMEOUT to 20 seconds. --- src/program/snabbvmx/tests/nexthop/selftest.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index 3bc39fa7ed..f05756fecd 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -32,11 +32,6 @@ SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock GUEST_MEM=1024 -function last_32bit { - mac=$1 - echo `echo $mac | egrep -o "[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+$"` -} - function cleanup { exit $1 } @@ -56,23 +51,26 @@ if ! snabb $SNABB_PCI1 "packetblaster replay -D 10 $PCAP_INPUT/v4v6-256.pcap $SN fi # Query nexthop for 10 seconds. -TIMEOUT=10 +TIMEOUT=20 count=0 while true; do output=`./snabb snabbvmx nexthop | egrep -o "[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+"` mac_v4=`echo "$output" | head -1` mac_v6=`echo "$output" | tail -1` - # Somehow the first 16-bit of nexhop come from the VM corrupted, compare only last 32-bit. - if [[ $(last_32bit "$mac_v4") == "99:99:99:99" && - $(last_32bit "$mac_v6") == "99:99:99:99" ]]; then + # FIXME: Should return expected MAC addresses. + # Check VM returned something. + if [[ "$mac_v4" != "00:00:00:00:00:00" && + "$mac_v6" != "00:00:00:00:00:00" ]]; then echo "Resolved MAC inet side: $mac_v4 [OK]" - echo "Resolved MAC inet side: $mac_v6 [OK]" + echo "Resolved MAC b4 side: $mac_v6 [OK]" exit 0 fi if [[ $count == $TIMEOUT ]]; then echo "Could not resolve nexthop" + echo "MAC inet side: $mac_v4 [FAILED]" + echo "MAC b4 side: $mac_v6 [FAILED]" exit 1 fi From 2185e97ebc866070e9828fc23648b687f9963e38 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 4 Nov 2016 19:01:43 +0100 Subject: [PATCH 121/631] Rename counters to counters_path --- src/program/snabbvmx/tests/end-to-end/core-end-to-end.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/snabbvmx/tests/end-to-end/core-end-to-end.sh b/src/program/snabbvmx/tests/end-to-end/core-end-to-end.sh index 4aa47b4265..42f433ea47 100755 --- a/src/program/snabbvmx/tests/end-to-end/core-end-to-end.sh +++ b/src/program/snabbvmx/tests/end-to-end/core-end-to-end.sh @@ -37,7 +37,7 @@ function run_and_cmp { } function run_and_regen_counters { - conf=$1; v4_in=$2; v6_in=$3; v4_out=$4; v6_out=$5; counters=$6 + conf=$1; v4_in=$2; v6_in=$3; v4_out=$4; v6_out=$5; counters_path=$6 endoutv4="${TEST_OUT}/endoutv4.pcap"; endoutv6="${TEST_OUT}/endoutv6.pcap"; rm -f $endoutv4 $endoutv6 ${SNABB_LWAFTR} check -r \ From 90add3d291a8158a247ea761c6b319bbd334c7e3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 09:33:37 +0100 Subject: [PATCH 122/631] Fix lib.yang.data serialization selftest --- src/lib/yang/data.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 6f2e7eb6dd..8670918d58 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -520,7 +520,7 @@ function selftest() print_data_for_schema(test_schema, data, file) file:close() local file = io.open(tmp, 'r') - local data = load_data_for_schema(test_schema, file:read('*a'), tmp) + data = load_data_for_schema(test_schema, file:read('*a'), tmp) file:close() os.remove(tmp) end From e1332fdaa669e3a87078e890670dbdcacde834dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sat, 5 Nov 2016 13:09:13 +0100 Subject: [PATCH 123/631] Optionally disable openstack support in Snabb --- default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index ab7faf4cbf..f2536f0e2a 100644 --- a/default.nix +++ b/default.nix @@ -5,6 +5,7 @@ { pkgs ? (import {}) , source ? ./. , version ? "dev" +, supportOpenstack ? true }: with pkgs; @@ -23,7 +24,7 @@ stdenv.mkDerivation rec { for f in $(find src/program/snabbnfv/ -type f); do substituteInPlace $f --replace "/bin/bash" "${bash}/bin/bash" done - + '' + lib.optionalString supportOpenstack '' # We need a way to pass $PATH to the scripts sed -i '2iexport PATH=${git}/bin:${mariadb}/bin:${which}/bin:${procps}/bin:${coreutils}/bin' src/program/snabbnfv/neutron_sync_master/neutron_sync_master.sh.inc sed -i '2iexport PATH=${git}/bin:${coreutils}/bin:${diffutils}/bin:${nettools}/bin' src/program/snabbnfv/neutron_sync_agent/neutron_sync_agent.sh.inc From 66903b4af15394c02db502de5d39e6a59e8ecdfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 7 Nov 2016 10:13:39 +0100 Subject: [PATCH 124/631] lwaftr: README.virtualization.md: fix formatting --- src/program/lwaftr/doc/README.virtualization.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/program/lwaftr/doc/README.virtualization.md b/src/program/lwaftr/doc/README.virtualization.md index 7e3180ebdd..6d316d7b0c 100644 --- a/src/program/lwaftr/doc/README.virtualization.md +++ b/src/program/lwaftr/doc/README.virtualization.md @@ -183,8 +183,6 @@ There is a screen on: Attach to this screen to see the RX/TX statistics of the runnign lwAFTR process. -``` - # Stop lwAFTR in a guest All the listed command so far can take a `stop` action that will stop them running. From 391a353a7c698a090141b6511538db7c90626ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 7 Nov 2016 10:14:36 +0100 Subject: [PATCH 125/631] lwaftrctl: pass $SSH_OPTS to all ssh invocations When benchmarking lwaftr using Nix, we have to pass custom private ssh key file to be able to login into the VM. --- src/program/lwaftr/virt/lwaftrctl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index d0364acd66..4c5f2c293b 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -61,7 +61,7 @@ start_vm() { stop_vm() { echo "Switching off VM. Please wait..." - ssh ${VM_USER}@${VM_IP} 'sudo halt' + ssh $SSH_OPTS ${VM_USER}@${VM_IP} 'sudo halt' sleep 10 pid=`pidof qemu-system-x86_64` sudo kill ${pid} 2>/dev/null @@ -76,13 +76,13 @@ restart_vm() { start_lwaftr() { echo "Start lwAFTR" - ssh ${VM_USER}@${VM_IP} "~/run_lwaftr.sh" + ssh $SSH_OPTS${VM_USER}@${VM_IP} "~/run_lwaftr.sh" } stop_lwaftr() { echo "Stop lwAFTR" - ssh ${VM_USER}@${VM_IP} 'sudo killall snabb 2>/dev/null' + ssh $SSH_OPTS ${VM_USER}@${VM_IP} 'sudo killall snabb 2>/dev/null' } restart_lwaftr() { From b375f38475bb8198a77dfa2c7189e134ad4008be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 7 Nov 2016 10:16:42 +0100 Subject: [PATCH 126/631] lwaftrctl: invoke screen with -L to enable logging Useful for post-mortem analysis. --- src/program/lwaftr/virt/lwaftrctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index 4c5f2c293b..aab2bc28be 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -108,7 +108,7 @@ start_snabbnfv_process() { fi # Run snabbnfv inside screen. - screen -dmS $screen bash -c \ + screen -L -dmS $screen bash -c \ "cd ${SNABB_HOST_PATH}/src; \ sudo numactl -m ${NUMA_NODE} taskset -c ${core} sudo ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock};" sleep 0.5 From 823776f119e0ee55df2442c28c36019da3dc8e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 7 Nov 2016 10:19:43 +0100 Subject: [PATCH 127/631] lwaftrctl: avoid double sudo In Nix, sudo doesn't have sudo executable available as it's not needed. --- src/program/lwaftr/virt/lwaftrctl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index aab2bc28be..20428ddeea 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -110,14 +110,14 @@ start_snabbnfv_process() { # Run snabbnfv inside screen. screen -L -dmS $screen bash -c \ "cd ${SNABB_HOST_PATH}/src; \ - sudo numactl -m ${NUMA_NODE} taskset -c ${core} sudo ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock};" + sudo numactl -m ${NUMA_NODE} taskset -c ${core} ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock};" sleep 0.5 status=$(screen -ls | grep ${screen}) # Check exit status. if [[ -z ${status} ]]; then echo "Start of snabbnfv failed: " - echo -e "\tsudo numactl -m ${NUMA_NODE} taskset -c ${core} sudo ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock}" + echo -e "\tsudo numactl -m ${NUMA_NODE} taskset -c ${core} ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock}" exit 1 fi From 387f99f4959a3edc071259fa083438a3c9c0e483 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 11:10:14 +0100 Subject: [PATCH 128/631] snabb-softwire: ICMP error limiting on both sides --- src/lib/yang/snabb-softwire-v1.yang | 48 ++++++++++++++++------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 1047a2d7f9..c5b0d3be84 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -75,6 +75,30 @@ module snabb-softwire-v1 { } } + grouping error-rate-limiting { + description + "These settings limit the rate of ICMP error message + transmission."; + + container error-rate-limiting { + presence true; + + leaf packets { + type uint32; + description + "The number of ICMP error messages which can be sent within + the specified time period."; + } + + leaf period { + type uint32 { range 1..max; } + default 2; + description + "The time period given in seconds."; + } + } + } + container external-interface { description "Configuration for the external, internet-facing IPv4 @@ -106,6 +130,7 @@ module snabb-softwire-v1 { using traffic-filters; using icmp-policy; using vlan-tagging; + using error-rate-limiting; container next-hop { presence true; @@ -154,6 +179,7 @@ module snabb-softwire-v1 { using traffic-filters; using icmp-policy; using vlan-tagging; + using error-rate-limiting; container next-hop { presence true; @@ -179,28 +205,6 @@ module snabb-softwire-v1 { two B4s."; } } - - container error-rate-limiting { - description - "These settings limit the rate of ICMP error message - transmission."; - - presence true; - - leaf packets { - type uint32; - description - "The number of ICMP error messages which can be sent within - the specified time period."; - } - - leaf period { - type uint32 { range 1..max; } - default 2; - description - "The time period given in seconds."; - } - } } container binding-table { From c0370a54afd429fa6d4cc3cdf00db63747e9f0cd Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 7 Nov 2016 11:19:35 +0100 Subject: [PATCH 129/631] Add `snabb config derive-data-format` option This introduces the config program and adds an option to produce an example of what the data format should look like based upon a valid YANG schema. --- src/program/config/README.inc | 2 + src/program/config/config.lua | 136 ++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 src/program/config/README.inc create mode 100644 src/program/config/config.lua diff --git a/src/program/config/README.inc b/src/program/config/README.inc new file mode 100644 index 0000000000..ca3ccad08d --- /dev/null +++ b/src/program/config/README.inc @@ -0,0 +1,2 @@ +Usage: + config derive-data-format [OPTIONS] [] diff --git a/src/program/config/config.lua b/src/program/config/config.lua new file mode 100644 index 0000000000..c6568736e0 --- /dev/null +++ b/src/program/config/config.lua @@ -0,0 +1,136 @@ +module(..., package.seeall) + +local lib = require("core.lib") +local usage = require("program.config.README_inc") +local schema = require("lib.yang.schema") +local yang_data = require("lib.yang.data") + +local tab_spaces = 4 + +function derive_data_format(a) + function print_level(level, ...) + io.write(string.rep(" ", level * tab_spaces)) + print(...) + end + function union_type(union) + local rtn + for _, t in pairs(union.argument_type.union) do + if rtn then + rtn = rtn .. " | " .. t.argument_string + else + rtn = t.argument_string + end + end + return rtn + end + function collate_options(name, node, keys) + local options = {} + if keys ~= nil then + for _, k in pairs(keys) do + if name == k then options.key = true end + end + end + if node.argument_type then + local at = node.argument_type + if at.range then + options.range = at.range.argument_string + end + end + if node.mandatory then + options.mandatory = true + end + return options + end + function display_node(name, node, options, level, parent) + function comment() + local comments = {} + if options.mandatory == true then + comments[#comments + 1] = "mandatory" + end + if options.key then + comments[#comments + 1] = "key" + end + if options.range then + comments[#comments + 1] = "between " .. options.range + end + local rtn = nil + for n, c in pairs(comments) do + if n == 1 then + rtn = "// " .. c + else + rtn = rtn .. " " .. c + end + end + return rtn + end + function display_leaf(keyword, argument) + if argument == "union" then argument = union_type(node) end + local comments = comment() + local str = keyword .. " " .. argument .. ";" + if comments then + print_level(level, str .. " " .. comments) + else + print_level(level, str) + end + end + if level == nil then level = 0 end + if node.type == "table" or node.type == "struct" then + local keys = {} + if node.keys then + print_level( + level, + "// List: this nested structure is repeated with (a) unique key(s)" + ) + end + print_level(level, name.." {") + display_data_format(node, level + 1) + print_level(level, "}") + elseif node.type == "scalar" then + display_leaf(name, node.argument_type.argument_string) + elseif node.type == "array" then + print_level( + level, + "// Array: made by repeating the keyword followed by each element" + ) + display_leaf(name, node.element_type.argument_string) + else + end + end + + function display_data_format(grammar, level) + for k,v in pairs(grammar.members) do + local options = collate_options(k, v, grammar.keys) + display_node(k, v, options, level) + end + end + + if #a <= 1 then + print(usage) + main.exit(1) + end + + local module = a[2] + + -- Fetch and parse the schema module. + local s = schema.parse_schema_file(module) + local grammar = yang_data.data_grammar_from_schema(s) + + display_data_format(grammar, 0) + +end + +function run(args) + local options = {} + options["derive-data-format"] = derive_data_format + + if #args <= 0 then + print(usage) + main.exit(1) + elseif options[args[1]] == nil then + print("Unknown argument: "..args[1]) + print(usage) + main.exit(1) + else + options[args[1]](args) + end +end From 9f57b65e248a2c4805dfec99b7a1f3718ded086c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 11:11:18 +0100 Subject: [PATCH 130/631] snabb-softwire: Add reassembly limits --- src/lib/yang/snabb-softwire-v1.yang | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index c5b0d3be84..695904ea3a 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -99,6 +99,38 @@ module snabb-softwire-v1 { } } + grouping reassembly { + description + "These settings limit the resources devoted to reassembling + fragmented packets."; + + container reassembly { + presence true; + + leaf max-fragments-per-packet { + type uint32 { range 1..max; } + default 20; + description + "The maximum number of fragments per reassembled packet. + Attempts to reassemble a packet using more fragments than + this threshold will fail and the reassembly data will be + discarded."; + } + + leaf max-packets { + type uint32; + default 20000; + description + "The maximum number of concurrent reassembly attempts. If + this limit is reached, an additional reassembly will cause + random eviction of an ongoing reassembly. Note that this + setting directly affects memory usage; the memory buffer + allocated to reassembly is this maximum number of + reassemblies times 25 kilobytes each."; + } + } + } + container external-interface { description "Configuration for the external, internet-facing IPv4 @@ -131,6 +163,7 @@ module snabb-softwire-v1 { using icmp-policy; using vlan-tagging; using error-rate-limiting; + using reassembly; container next-hop { presence true; @@ -180,6 +213,7 @@ module snabb-softwire-v1 { using icmp-policy; using vlan-tagging; using error-rate-limiting; + using reassembly; container next-hop { presence true; From cb1e4d784b08763a9c02e57265081f5c222d4153 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 12:07:50 +0100 Subject: [PATCH 131/631] snabb-softwire: Change softwire leaf members The new names match the ones that the source code expects. Weird, right? --- src/lib/yang/snabb-softwire-v1.yang | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 695904ea3a..6942fb0dd4 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -339,9 +339,9 @@ module snabb-softwire-v1 { } list softwire { - key "ipv4-address psid padding"; + key "ipv4 psid padding"; - leaf ipv4-address { + leaf ipv4 { type inet:ipv4-address; mandatory true; description @@ -370,7 +370,7 @@ module snabb-softwire-v1 { a zero-based index into br-addresses."; } - leaf ipv6-address { + leaf b4-ipv6 { type inet:ipv6-address; mandatory true; description From 5f7745826567982be056ba753c3ec277f65ee59a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 12:22:04 +0100 Subject: [PATCH 132/631] Add another YANG data representation test case --- src/lib/yang/binary.lua | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 32b47eee0d..fded6fb960 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -375,6 +375,7 @@ function selftest() leaf is-active { type boolean; default true; } leaf-list integers { type uint32; } + leaf-list addrs { type inet:ipv4-address; } container routes { presence true; list route { @@ -389,6 +390,8 @@ function selftest() integers 1; integers 2; integers 0xffffffff; + addrs 4.3.2.1; + addrs 5.4.3.2; routes { route { addr 1.2.3.4; port 1; } route { addr 2.3.4.5; port 10; } @@ -398,12 +401,18 @@ function selftest() local ipv4 = require('lib.protocol.ipv4') + local function uint32_ptr(addr) + return ffi.cast('uint32_t*', addr) + end for i=1,3 do assert(data.is_active == true) assert(#data.integers == 3) assert(data.integers[1] == 1) assert(data.integers[2] == 2) assert(data.integers[3] == 0xffffffff) + assert(#data.addrs == 2) + assert(uint32_ptr(data.addrs[1])[0]==uint32_ptr(ipv4:pton('4.3.2.1'))[0]) + assert(uint32_ptr(data.addrs[2])[0]==uint32_ptr(ipv4:pton('5.4.3.2'))[0]) local routing_table = data.routes.route assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) From f93f1b6cf49c8f6617dfd33e0fbd87e566218a37 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 7 Nov 2016 12:42:19 +0100 Subject: [PATCH 133/631] Move data_format into own directory and rename This relocates the command "derive-data-format" into it's own directory to make it cleaner and easier to add new commands to the `snabb config` program. This also renames it from "derive-data-format" to simply be "data-format". --- src/program/config/config.lua | 131 +----------------- src/program/config/data_format/README.inc | 27 ++++ .../config/data_format/data_format.lua | 119 ++++++++++++++++ 3 files changed, 151 insertions(+), 126 deletions(-) create mode 100644 src/program/config/data_format/README.inc create mode 100644 src/program/config/data_format/data_format.lua diff --git a/src/program/config/config.lua b/src/program/config/config.lua index c6568736e0..fa49be5167 100644 --- a/src/program/config/config.lua +++ b/src/program/config/config.lua @@ -1,136 +1,15 @@ module(..., package.seeall) -local lib = require("core.lib") local usage = require("program.config.README_inc") -local schema = require("lib.yang.schema") -local yang_data = require("lib.yang.data") - -local tab_spaces = 4 - -function derive_data_format(a) - function print_level(level, ...) - io.write(string.rep(" ", level * tab_spaces)) - print(...) - end - function union_type(union) - local rtn - for _, t in pairs(union.argument_type.union) do - if rtn then - rtn = rtn .. " | " .. t.argument_string - else - rtn = t.argument_string - end - end - return rtn - end - function collate_options(name, node, keys) - local options = {} - if keys ~= nil then - for _, k in pairs(keys) do - if name == k then options.key = true end - end - end - if node.argument_type then - local at = node.argument_type - if at.range then - options.range = at.range.argument_string - end - end - if node.mandatory then - options.mandatory = true - end - return options - end - function display_node(name, node, options, level, parent) - function comment() - local comments = {} - if options.mandatory == true then - comments[#comments + 1] = "mandatory" - end - if options.key then - comments[#comments + 1] = "key" - end - if options.range then - comments[#comments + 1] = "between " .. options.range - end - local rtn = nil - for n, c in pairs(comments) do - if n == 1 then - rtn = "// " .. c - else - rtn = rtn .. " " .. c - end - end - return rtn - end - function display_leaf(keyword, argument) - if argument == "union" then argument = union_type(node) end - local comments = comment() - local str = keyword .. " " .. argument .. ";" - if comments then - print_level(level, str .. " " .. comments) - else - print_level(level, str) - end - end - if level == nil then level = 0 end - if node.type == "table" or node.type == "struct" then - local keys = {} - if node.keys then - print_level( - level, - "// List: this nested structure is repeated with (a) unique key(s)" - ) - end - print_level(level, name.." {") - display_data_format(node, level + 1) - print_level(level, "}") - elseif node.type == "scalar" then - display_leaf(name, node.argument_type.argument_string) - elseif node.type == "array" then - print_level( - level, - "// Array: made by repeating the keyword followed by each element" - ) - display_leaf(name, node.element_type.argument_string) - else - end - end - - function display_data_format(grammar, level) - for k,v in pairs(grammar.members) do - local options = collate_options(k, v, grammar.keys) - display_node(k, v, options, level) - end - end - - if #a <= 1 then - print(usage) - main.exit(1) - end - - local module = a[2] - - -- Fetch and parse the schema module. - local s = schema.parse_schema_file(module) - local grammar = yang_data.data_grammar_from_schema(s) - - display_data_format(grammar, 0) - -end function run(args) - local options = {} - options["derive-data-format"] = derive_data_format - + -- Display usage if we have no arguments. if #args <= 0 then print(usage) main.exit(1) - elseif options[args[1]] == nil then - print("Unknown argument: "..args[1]) - print(usage) - main.exit(1) - else - options[args[1]](args) end + + local command = string.gsub(table.remove(args, 1), "-", "_") + local modname = ("program.config.%s.%s"):format(command, command) + require(modname).run(args) end diff --git a/src/program/config/data_format/README.inc b/src/program/config/data_format/README.inc new file mode 100644 index 0000000000..44f22c7567 --- /dev/null +++ b/src/program/config/data_format/README.inc @@ -0,0 +1,27 @@ +Data Format Usage: + snabb config data-format + +This command produces an annotated yang data format from a schema file +which can be used to check a yang schema or help you write the data +configuration file. + +The output is an option or a container of options followed by the type +of value that is to be expected. In the case where there is a option +with a section in curly braces that will represent a nested structure +such as a list or a container. Comments preciding certain fields will +indicate if for example the block is a list, or comments may proceed +options specifying if they are for example mandatory. + +An exaaple of an option could be: + + port uint8; // mandatory between 0..11 + +This describes a configuration option "port" which takes an unsigned +integer with a value between 0 and 11. The option is required so must +have a value. An example for this field in the configuration could be: + + port 7; + +Example usage: + + $ snabb config data-format lib/yang/snabb-softwire.yang diff --git a/src/program/config/data_format/data_format.lua b/src/program/config/data_format/data_format.lua new file mode 100644 index 0000000000..1dee961e1e --- /dev/null +++ b/src/program/config/data_format/data_format.lua @@ -0,0 +1,119 @@ +module(..., package.seeall) + +local schema = require("lib.yang.schema") +local yang_data = require("lib.yang.data") +local usage = require("program.config.data_format.README_inc") + +-- Number of spaces a tab should consist of when indenting config. +local tab_spaces = 4 + +function run(args) + function print_level(level, ...) + io.write(string.rep(" ", level * tab_spaces)) + print(...) + end + function union_type(union) + local rtn + for _, t in pairs(union.argument_type.union) do + if rtn then + rtn = rtn .. " | " .. t.argument_string + else + rtn = t.argument_string + end + end + return rtn + end + function collate_options(name, node, keys) + local options = {} + if keys ~= nil then + for _, k in pairs(keys) do + if name == k then options.key = true end + end + end + if node.argument_type then + local at = node.argument_type + if at.range then + options.range = at.range.argument_string + end + end + if node.mandatory then + options.mandatory = true + end + return options + end + function display_node(name, node, options, level, parent) + function comment() + local comments = {} + if options.mandatory == true then + comments[#comments + 1] = "mandatory" + end + if options.key then + comments[#comments + 1] = "key" + end + if options.range then + comments[#comments + 1] = "between " .. options.range + end + local rtn = nil + for n, c in pairs(comments) do + if n == 1 then + rtn = "// " .. c + else + rtn = rtn .. " " .. c + end + end + return rtn + end + function display_leaf(keyword, argument) + if argument == "union" then argument = union_type(node) end + local comments = comment() + local str = keyword .. " " .. argument .. ";" + if comments then + print_level(level, str .. " " .. comments) + else + print_level(level, str) + end + end + if level == nil then level = 0 end + if node.type == "table" or node.type == "struct" then + local keys = {} + if node.keys then + print_level( + level, + "// List: this nested structure is repeated with (a) unique key(s)" + ) + end + print_level(level, name.." {") + display_data_format(node, level + 1) + print_level(level, "}") + elseif node.type == "scalar" then + display_leaf(name, node.argument_type.argument_string) + elseif node.type == "array" then + print_level( + level, + "// Array: made by repeating the keyword followed by each element" + ) + display_leaf(name, node.element_type.argument_string) + else + end + end + + function display_data_format(grammar, level) + for k,v in pairs(grammar.members) do + local options = collate_options(k, v, grammar.keys) + display_node(k, v, options, level) + end + end + + if #args <= 0 then + print(usage) + main.exit(1) + end + + local yangmod = args[1] + + -- Fetch and parse the schema module. + local s = schema.parse_schema_file(yangmod) + local grammar = yang_data.data_grammar_from_schema(s) + + display_data_format(grammar, 0) +end From 34871fb29345ef35a0bc6a780faf0db5329560b7 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 7 Nov 2016 13:42:45 +0100 Subject: [PATCH 134/631] Documentation fixes, suggested by Christian Graf --- src/program/snabbvmx/doc/README.troubleshooting.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/program/snabbvmx/doc/README.troubleshooting.md b/src/program/snabbvmx/doc/README.troubleshooting.md index c6e177049d..27de9682d1 100644 --- a/src/program/snabbvmx/doc/README.troubleshooting.md +++ b/src/program/snabbvmx/doc/README.troubleshooting.md @@ -382,8 +382,8 @@ decapsulated, etc), you craft a hand-made packet that meets the testing case. Now we can check what the lwAFTR produces: ```bash -sudo ./snabb snabbvmx -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap empty.pcap \ - /tmp/outv4.pcap /tmp/outv6.pcap counters.lua +sudo ./snabb snabbvmx check -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap \ + empty.pcap /tmp/outv4.pcap /tmp/outv6.pcap counters.lua ``` The flag `-r` generates a counters file. @@ -391,7 +391,7 @@ The flag `-r` generates a counters file. Check that your output matches what you expect: ```bash -$ tcpdump -qns 0 -ter empty.pcap +$ tcpdump -qns 0 -ter /tmp/outv4.pcap reading from file empty.pcap, link-type EN10MB (Ethernet) ``` From 1594d20fcf0d4b3de84873e25a97924bd61a805f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 7 Nov 2016 14:21:11 +0100 Subject: [PATCH 135/631] lwaftrctl: typo --- src/program/lwaftr/virt/lwaftrctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index 20428ddeea..753d407875 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -76,7 +76,7 @@ restart_vm() { start_lwaftr() { echo "Start lwAFTR" - ssh $SSH_OPTS${VM_USER}@${VM_IP} "~/run_lwaftr.sh" + ssh $SSH_OPTS ${VM_USER}@${VM_IP} "~/run_lwaftr.sh" } stop_lwaftr() { From fc75a9926cb0517f57c568cd5376e577d532ebf1 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 7 Nov 2016 14:37:00 +0100 Subject: [PATCH 136/631] Use core.lib.dogetopt to support -h and --help This adds flag support for both `snabb config` and the `snabb config data-format` sub-program. They now both respond appropriately to the "-h" and "--help" flags by displaying the appropriate help information. --- src/program/config/config.lua | 23 +++++++++++++------ .../config/data_format/data_format.lua | 23 ++++++++++++++----- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/program/config/config.lua b/src/program/config/config.lua index fa49be5167..6acd10fad1 100644 --- a/src/program/config/config.lua +++ b/src/program/config/config.lua @@ -1,14 +1,23 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -local usage = require("program.config.README_inc") +local lib = require("core.lib") -function run(args) - -- Display usage if we have no arguments. - if #args <= 0 then - print(usage) - main.exit(1) - end +local function show_usage(status) + print(require("program.config.README_inc")) + main.exit(status) +end +local function parse_args(args) + local handlers = {} + handlers.h = function() show_usage(0) end + args = lib.dogetopt(args, handlers, "h", {help="h"}) + if #args <= 1 then show_usage(1) end + return args +end + +function run(args) + args = parse_args(args) local command = string.gsub(table.remove(args, 1), "-", "_") local modname = ("program.config.%s.%s"):format(command, command) require(modname).run(args) diff --git a/src/program/config/data_format/data_format.lua b/src/program/config/data_format/data_format.lua index 1dee961e1e..c0a0c666de 100644 --- a/src/program/config/data_format/data_format.lua +++ b/src/program/config/data_format/data_format.lua @@ -1,5 +1,7 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) +local lib = require("core.lib") local schema = require("lib.yang.schema") local yang_data = require("lib.yang.data") local usage = require("program.config.data_format.README_inc") @@ -7,6 +9,19 @@ local usage = require("program.config.data_format.README_inc") -- Number of spaces a tab should consist of when indenting config. local tab_spaces = 4 +local function show_usage(status) + print(require("program.config.data_format.README_inc")) + main.exit(status) +end + +local function parse_args(args) + local handlers = {} + handlers.h = function() show_usage(0) end + args = lib.dogetopt(args, handlers, "h", {help="h"}) + if #args <= 0 then show_usage(1) end + return unpack(args) +end + function run(args) function print_level(level, ...) io.write(string.rep(" ", level * tab_spaces)) @@ -104,15 +119,11 @@ function run(args) end end - if #args <= 0 then - print(usage) - main.exit(1) - end - local yangmod = args[1] + local yang_module = parse_args(args) -- Fetch and parse the schema module. - local s = schema.parse_schema_file(yangmod) + local s = schema.parse_schema_file(yang_module) local grammar = yang_data.data_grammar_from_schema(s) display_data_format(grammar, 0) From 66d1cddb40f73b4c29e349f2660067502cecf796 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 7 Nov 2016 15:23:00 +0100 Subject: [PATCH 137/631] Convert node displaying into data driven code This converts a if-elseif-else block which was being used as a mechanism for selecting between different node types and instead converting it to a more function driven method. Now there is a table of methods built up that handle the different types and those are selected based of the node type to handle the display of the data. --- .../config/data_format/data_format.lua | 117 +++++++++--------- 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/src/program/config/data_format/data_format.lua b/src/program/config/data_format/data_format.lua index c0a0c666de..8764ee2b0c 100644 --- a/src/program/config/data_format/data_format.lua +++ b/src/program/config/data_format/data_format.lua @@ -38,10 +38,10 @@ function run(args) end return rtn end - function collate_options(name, node, keys) + function collate_options(name, node) local options = {} - if keys ~= nil then - for _, k in pairs(keys) do + if node.keys ~= nil then + for _, k in pairs(node.keys) do if name == k then options.key = true end end end @@ -56,75 +56,80 @@ function run(args) end return options end - function display_node(name, node, options, level, parent) - function comment() - local comments = {} - if options.mandatory == true then - comments[#comments + 1] = "mandatory" - end - if options.key then - comments[#comments + 1] = "key" - end - if options.range then - comments[#comments + 1] = "between " .. options.range - end - local rtn = nil - for n, c in pairs(comments) do - if n == 1 then - rtn = "// " .. c - else - rtn = rtn .. " " .. c - end - end - return rtn + function comment(options) + local comments = {} + if options.mandatory == true then + comments[#comments + 1] = "mandatory" + end + if options.key then + comments[#comments + 1] = "key" + end + if options.range then + comments[#comments + 1] = "between " .. options.range end - function display_leaf(keyword, argument) - if argument == "union" then argument = union_type(node) end - local comments = comment() - local str = keyword .. " " .. argument .. ";" - if comments then - print_level(level, str .. " " .. comments) + local rtn = nil + for n, c in pairs(comments) do + if n == 1 then + rtn = "// " .. c else - print_level(level, str) + rtn = rtn .. " " .. c end end + return rtn + end + + function display_leaf(level, keyword, argument, options) + if argument == "union" then argument = union_type(node) end + local comments = comment(options) + local str = keyword .. " ".. argument .. ";" + if comments then + print_level(level, str .. " " .. comments) + else + print_level(level, str) + end + end + + local describers = {} + local function describe(level, name, node, options) + local err = "Unknown node type: "..node.type + local options = collate_options(name, node) + assert(describers[node.type], err)(level, name, node, options) + end + function describe_members(node, level) if level == nil then level = 0 end - if node.type == "table" or node.type == "struct" then - local keys = {} - if node.keys then - print_level( - level, - "// List: this nested structure is repeated with (a) unique key(s)" - ) - end - print_level(level, name.." {") - display_data_format(node, level + 1) - print_level(level, "}") - elseif node.type == "scalar" then - display_leaf(name, node.argument_type.argument_string) - elseif node.type == "array" then + for name, n in pairs(node.members) do + local options = collate_options(name, node) + describe(level, name, n, options) + end + end + function describers.scalar(level, name, node, options) + display_leaf(level, name, node.argument_type.argument_string, options) + end + function describers.table(level, name, node, options) + if node.keys then print_level( level, - "// Array: made by repeating the keyword followed by each element" + "// List: this nested structure is repeated with (a) unique key(s)" ) - display_leaf(name, node.element_type.argument_string) - else end + print_level(level, name.." {") + describe_members(node, level + 1) + print_level(level, "}") end - - function display_data_format(grammar, level) - for k,v in pairs(grammar.members) do - local options = collate_options(k, v, grammar.keys) - display_node(k, v, options, level) - end + describers.struct = describers.table + function describers.array(level, name, node, options) + print_level( + level, + "// Array: made by repeating the keyword followed by each element" + ) + display_leaf(level, name, node.element_type.argument_string, options) end - local yang_module = parse_args(args) -- Fetch and parse the schema module. local s = schema.parse_schema_file(yang_module) local grammar = yang_data.data_grammar_from_schema(s) - display_data_format(grammar, 0) + describe_members(grammar) end From f659341c06ad8a104b4ac51b1eb2f18738ec3ab4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 15:39:17 +0100 Subject: [PATCH 138/631] lib.yang IPv4 representation is host-endian uint32 --- src/lib/yang/binary.lua | 17 +++++++++-------- src/lib/yang/data.lua | 2 +- src/lib/yang/util.lua | 23 ++++++++++++++++------- src/lib/yang/value.lua | 17 +++++------------ 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index fded6fb960..0bef3b70b6 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -401,9 +401,6 @@ function selftest() local ipv4 = require('lib.protocol.ipv4') - local function uint32_ptr(addr) - return ffi.cast('uint32_t*', addr) - end for i=1,3 do assert(data.is_active == true) assert(#data.integers == 3) @@ -411,12 +408,16 @@ function selftest() assert(data.integers[2] == 2) assert(data.integers[3] == 0xffffffff) assert(#data.addrs == 2) - assert(uint32_ptr(data.addrs[1])[0]==uint32_ptr(ipv4:pton('4.3.2.1'))[0]) - assert(uint32_ptr(data.addrs[2])[0]==uint32_ptr(ipv4:pton('5.4.3.2'))[0]) + assert(data.addrs[1]==util.ipv4_pton('4.3.2.1')) + assert(data.addrs[2]==util.ipv4_pton('5.4.3.2')) local routing_table = data.routes.route - assert(routing_table:lookup_ptr(ipv4:pton('1.2.3.4')).value.port == 1) - assert(routing_table:lookup_ptr(ipv4:pton('2.3.4.5')).value.port == 10) - assert(routing_table:lookup_ptr(ipv4:pton('3.4.5.6')).value.port == 2) + local key = ffi.new('struct { uint32_t addr; }') + key.addr = util.ipv4_pton('1.2.3.4') + assert(routing_table:lookup_ptr(key).value.port == 1) + key.addr = util.ipv4_pton('2.3.4.5') + assert(routing_table:lookup_ptr(key).value.port == 10) + key.addr = util.ipv4_pton('3.4.5.6') + assert(routing_table:lookup_ptr(key).value.port == 2) local tmp = os.tmpname() compile_data_for_schema(test_schema, data, tmp) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 8670918d58..34a1c0d25f 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -513,7 +513,7 @@ function selftest() assert(contents.bar.tree_grown == nil) assert(contents.baz.score == 9) assert(contents.baz.tree_grown == true) - assert(require('lib.protocol.ipv4'):ntop(data.addr) == '1.2.3.4') + assert(data.addr == util.ipv4_pton('1.2.3.4')) local tmp = os.tmpname() local file = io.open(tmp, 'w') diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 955678c370..afe06c66f9 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -1,14 +1,9 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) +local lib = require("core.lib") local ffi = require("ffi") - --- Parse inet:mac-address using ethernet:pton --- Parse inet:ipv4-address using ipv4:pton --- Parse inet:ipv6-address using ipv6:pton - --- Parse inet:ipv4-prefix? --- Parse inet:ipv6-prefix? +local ipv4 = require("lib.protocol.ipv4") ffi.cdef([[ unsigned long long strtoull (const char *nptr, const char **endptr, int base); @@ -70,7 +65,18 @@ function ffi_array(ptr, elt_t, count) return ffi.metatype(ffi.typeof('struct { $* ptr; }', elt_t), mt)(ptr) end +-- The yang modules represent IPv4 addresses as host-endian uint32 +-- values in Lua. See https://github.com/snabbco/snabb/issues/1063. +function ipv4_pton(str) + return lib.ntohl(ffi.cast('uint32_t*', assert(ipv4:pton(str)))[0]) +end + +function ipv4_ntop(addr) + return ipv4:ntop(ffi.new('uint32_t[1]', lib.htonl(addr))) +end + function selftest() + print('selftest: lib.yang.util') assert(tointeger('0') == 0) assert(tointeger('-0') == 0) assert(tointeger('10') == 10) @@ -83,4 +89,7 @@ function selftest() assert(tointeger('0xffffffffffffffff') == 0xffffffffffffffffULL) assert(tointeger('-0x7fffffffffffffff') == -0x7fffffffffffffffLL) assert(tointeger('-0x8000000000000000') == -0x8000000000000000LL) + assert(ipv4_pton('255.0.0.1') == 255 * 2^24 + 1) + assert(ipv4_ntop(ipv4_pton('255.0.0.1')) == '255.0.0.1') + print('selftest: ok') end diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua index e8cef2c8d6..43eca86bbc 100644 --- a/src/lib/yang/value.lua +++ b/src/lib/yang/value.lua @@ -9,13 +9,6 @@ local ffi = require("ffi") local bit = require("bit") local ethernet = require("lib.protocol.ethernet") --- FIXME: --- Parse inet:mac-address using ethernet:pton --- Parse inet:ipv4-address using ipv4:pton --- Parse inet:ipv6-address using ipv6:pton --- Parse inet:ipv4-prefix? --- Parse inet:ipv6-prefix? - types = {} local function integer_type(ctype) @@ -98,9 +91,9 @@ end types.union = unimplemented('union') types['ipv4-address'] = { - ctype = 'uint8_t[4]', - parse = function(str, what) return assert(ipv4:pton(str)) end, - tostring = function(val) return ipv4:ntop(val) end + ctype = 'uint32_t', + parse = function(str, what) return util.ipv4_pton(str) end, + tostring = function(val) return util.ipv4_ntop(val) end } types['ipv6-address'] = { @@ -119,9 +112,9 @@ types['ipv4-prefix'] = { ctype = 'struct { uint8_t[4] prefix; uint8_t len; }', parse = function(str, what) local prefix, len = str:match('^([^/]+)/(.*)$') - return { assert(ipv4:pton(prefix)), util.tointeger(len, 1, 32) } + return { ipv4_pton(prefix), util.tointeger(len, 1, 32) } end, - tostring = function(val) return ipv4:ntop(val[1])..'/'..tostring(val[2]) end + tostring = function(val) return ipv4_ntop(val[1])..'/'..tostring(val[2]) end } types['ipv6-prefix'] = { From 3c163de55c02d30c47cf4d4f75c5b11677d8b5fa Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 15:53:21 +0100 Subject: [PATCH 139/631] Add first pass of snabb lwaftr migrate-configuration Run like snabb lwaftr migrate-configuration program/lwaftr/tests/data/icmp_on_fail.conf Unfortunately some of the leaves are not serializing; need to fix. --- src/program/lwaftr/README | 1 + .../lwaftr/migrate_configuration/README | 11 ++ .../lwaftr/migrate_configuration/README.inc | 1 + .../migrate_configuration.lua | 120 ++++++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 src/program/lwaftr/migrate_configuration/README create mode 120000 src/program/lwaftr/migrate_configuration/README.inc create mode 100644 src/program/lwaftr/migrate_configuration/migrate_configuration.lua diff --git a/src/program/lwaftr/README b/src/program/lwaftr/README index 7851a18247..247fed4f38 100644 --- a/src/program/lwaftr/README +++ b/src/program/lwaftr/README @@ -5,6 +5,7 @@ Usage: snabb lwaftr transient snabb lwaftr compile-binding-table snabb lwaftr generate-binding-table + snabb lwaftr migrate-configuration snabb lwaftr control snabb lwaftr loadtest snabb lwaftr generator diff --git a/src/program/lwaftr/migrate_configuration/README b/src/program/lwaftr/migrate_configuration/README new file mode 100644 index 0000000000..0f5f9e33a7 --- /dev/null +++ b/src/program/lwaftr/migrate_configuration/README @@ -0,0 +1,11 @@ +Usage: migrate-configuration LWAFTR.CONF + +Options: + -h, --help Print usage information. + +Migrate an old-style configuration and binding table to the new YANG +configuration. LWAFTR.CONF should be the name of an old lwAFTR +configuration. + +The resulting new-style configuration will be printed on standard +output, ready to be saved to a new file. diff --git a/src/program/lwaftr/migrate_configuration/README.inc b/src/program/lwaftr/migrate_configuration/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/lwaftr/migrate_configuration/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua new file mode 100644 index 0000000000..1db8c6ae31 --- /dev/null +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -0,0 +1,120 @@ +module(..., package.seeall) + +local lib = require('core.lib') +local ffi = require('ffi') +local util = require('lib.yang.util') +local ipv4 = require('lib.protocol.ipv4') +local ctable = require('lib.ctable') +local binding_table = require("apps.lwaftr.binding_table") +local conf = require('apps.lwaftr.conf') +local load_lwaftr_config = conf.load_lwaftr_config +local ffi_array = require('lib.yang.util').ffi_array +local yang = require('lib.yang.yang') + +local function show_usage(code) + print(require("program.lwaftr.migrate_configuration.README_inc")) + main.exit(code) +end + +local function parse_args(args) + local handlers = {} + function handlers.h() show_usage(0) end + args = lib.dogetopt(args, handlers, "h", { help="h" }) + if #args ~= 1 then show_usage(1) end + return unpack(args) +end + +local function migrate_conf(old) + function convert_ipv4(addr) + if addr then return util.ipv4_pton(ipv4:ntop(addr)) end + end + local external = { + ip = convert_ipv4(old.aftr_ipv4_ip), + mac = old.aftr_mac_inet_side, + mtu = old.ipv4_mtu, + ingress_filter = old.ipv4_ingress_filter, + egress_filter = old.ipv4_egress_filter, + allow_incoming_icmp = old.policy_icmpv4_incoming == conf.policies.ALLOW, + generate_icmp_errors = old.policy_icmpv4_outgoing == conf.policies.ALLOW, + vlan_tag = old.v4_vlan_tag, + error_rate_limiting = { + packets = old.icmpv4_rate_limiter_n_packets, + seconds = old.icmpv4_rate_limiter_n_seconds + }, + reassembly = { + max_fragments_per_packet = old.max_fragments_per_reassembly_packet, + max_packets = old.max_ipv4_reassembly_packets + }, + next_hop = { + ip = convert_ipv4(old.next_hop_ipv4_addr), + mac = old.inet_mac + } + } + + local function pk(...) + print(...) + return ... + end + local internal = { + ip = old.aftr_ipv6_ip, + mac = old.aftr_mac_b4_side, + mtu = old.ipv6_mtu, + ingress_filter = old.ipv6_ingress_filter, + egress_filter = old.ipv6_egress_filter, + allow_incoming_icmp = old.policy_icmpv6_incoming == conf.policies.ALLOW, + generate_icmp_errors = old.policy_icmpv6_outgoing == conf.policies.ALLOW, + vlan_tag = old.v6_vlan_tag, + error_rate_limiting = { + packets = pk(old.icmpv6_rate_limiter_n_packets), + seconds = old.icmpv6_rate_limiter_n_seconds + }, + reassembly = { + max_fragments_per_packet = old.max_fragments_per_reassembly_packet, + max_packets = old.max_ipv6_reassembly_packets + }, + next_hop = { + ip = old.next_hop_ipv6_addr, + mac = old.next_hop6_mac + }, + hairpinning = old.hairpinning + } + + local old_bt = binding_table.load(old.binding_table) + local psid_map = {} + for addr, end_addr, params in old_bt.psid_map:iterate() do + local reserved_ports_bit_count = 16 - params.psid_length - params.shift + if end_addr == addr then end_addr = nil end + if reserved_ports_bit_count ~= 16 then + psid_map[{addr=addr}] = { + end_addr = end_addr, + psid_length = params.psid_length, + shift = params.shift, + reserved_ports_bit_count = reserved_ports_bit_count + } + end + end + local br_address_t = ffi.typeof('uint8_t[16]') + local br_address_array = ffi.cast (ffi.typeof('$*', br_address_t), + old_bt.br_addresses) + local br_addresses = ffi_array(br_address_array, br_address_t, + old_bt.br_address_count) + local softwires = old_bt.softwires + + return { + external_interface = external, + internal_interface = internal, + binding_table = { + psid_map = psid_map, + br_address = br_addresses, + softwire = softwires + } + } +end + +function run(args) + local conf_file = parse_args(args) + local old_conf = load_lwaftr_config(conf_file) + local new_conf = migrate_conf(old_conf) + yang.print_data_for_schema_by_name('snabb-softwire-v1', new_conf, io.stdout) + main.exit(0) +end From 23ff1d89c7897d3e1dd6a27a3b610c688f1c2388 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 16:19:06 +0100 Subject: [PATCH 140/631] Fix syntax error in snabb-softwire-v1.yang Doh, it's spelled "uses", not "using". We need to make our YANG schema parser do more validation! Also remove a debug print in migrate-configuration. --- src/lib/yang/snabb-softwire-v1.yang | 20 +++++++++---------- .../migrate_configuration.lua | 6 +----- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 6942fb0dd4..ddde2a1d64 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -159,11 +159,11 @@ module snabb-softwire-v1 { "Maximum packet size to send on the IPv4 interface."; } - using traffic-filters; - using icmp-policy; - using vlan-tagging; - using error-rate-limiting; - using reassembly; + uses traffic-filters; + uses icmp-policy; + uses vlan-tagging; + uses error-rate-limiting; + uses reassembly; container next-hop { presence true; @@ -209,11 +209,11 @@ module snabb-softwire-v1 { "Maximum packet size to sent on the IPv6 interface."; } - using traffic-filters; - using icmp-policy; - using vlan-tagging; - using error-rate-limiting; - using reassembly; + uses traffic-filters; + uses icmp-policy; + uses vlan-tagging; + uses error-rate-limiting; + uses reassembly; container next-hop { presence true; diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 1db8c6ae31..5bd1ecb84e 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -51,10 +51,6 @@ local function migrate_conf(old) } } - local function pk(...) - print(...) - return ... - end local internal = { ip = old.aftr_ipv6_ip, mac = old.aftr_mac_b4_side, @@ -65,7 +61,7 @@ local function migrate_conf(old) generate_icmp_errors = old.policy_icmpv6_outgoing == conf.policies.ALLOW, vlan_tag = old.v6_vlan_tag, error_rate_limiting = { - packets = pk(old.icmpv6_rate_limiter_n_packets), + packets = old.icmpv6_rate_limiter_n_packets, seconds = old.icmpv6_rate_limiter_n_seconds }, reassembly = { From 987c00863d96ad675028352b226270e042b017ce Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 7 Nov 2016 16:22:09 +0100 Subject: [PATCH 141/631] Migrate-configuration quiets binding table loading --- src/apps/lwaftr/binding_table.lua | 2 +- .../lwaftr/migrate_configuration/migrate_configuration.lua | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index f51e608fcc..e4ed9c2d3b 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -502,7 +502,7 @@ function load_source(text_stream) return parse_binding_table(Parser.new(text_stream)) end -local verbose = os.getenv('SNABB_LWAFTR_VERBOSE') or true +verbose = os.getenv('SNABB_LWAFTR_VERBOSE') or true local function log(msg, ...) if verbose then print(msg:format(...)) end end diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 5bd1ecb84e..f1f65edef3 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -108,6 +108,7 @@ local function migrate_conf(old) end function run(args) + binding_table.verbose = false local conf_file = parse_args(args) local old_conf = load_lwaftr_config(conf_file) local new_conf = migrate_conf(old_conf) From ac4b0ec467ac4739aa0c8f8996fa5940430bbf16 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 7 Nov 2016 16:56:37 +0100 Subject: [PATCH 142/631] Add Hydra CI docs (#545) Add Hydra CI docs, first draft --- .../doc/README.continuous-integration.md | 61 +++++++++++++++++++ src/program/lwaftr/doc/genbook.sh | 6 ++ 2 files changed, 67 insertions(+) create mode 100644 src/program/lwaftr/doc/README.continuous-integration.md diff --git a/src/program/lwaftr/doc/README.continuous-integration.md b/src/program/lwaftr/doc/README.continuous-integration.md new file mode 100644 index 0000000000..e7a2ad478f --- /dev/null +++ b/src/program/lwaftr/doc/README.continuous-integration.md @@ -0,0 +1,61 @@ +# Continuous integration + +The Snabb project has a continuous integration lab - the SnabbLab - described +in [the Snabb docs](http://snabbco.github.io/#snabblab). The lab servers are +used for checking correctness and performance of Snabb branches, before merging +them into the master branch. + +The servers are handled by [Hydra](https://nixos.org/hydra/), the continuous +integration system of the [NixOS](http://nixos.org/) operating system, which +the servers run. + +The code in the [snabblab-nixos](https://github.com/snabblab/snabblab-nixos) +repo defines the Hydra jobs that are run by [Snabb's Hydra instance] +(https://hydra.snabb.co/). It also defines the reports generated from the job +results. + +## lwAftr benchmarks + +[One of the projects](https://hydra.snabb.co/project/igalia) on Snabb's Hydra +instance hosts the lwAftr CI benchmarks. Three jobsets are currently defined: + +- [lwaftr-bare](https://hydra.snabb.co/jobset/igalia/lwaftr-bare) runs the + `snabb lwaftr bench` command, which executes lwAftr benchmarks with no + interaction with physical NICs; +- [lwaftr-nic](https://hydra.snabb.co/jobset/igalia/lwaftr-nic) runs the + `snabb lwaftr loadtest` and `snabb lwaftr run` commands: the first command + generates network traffic to a physical NIC, which is then received by + another NIC and processed by the lwAftr instance launched by the second + command; +- [lwaftr-virt](https://hydra.snabb.co/jobset/igalia/lwaftr-nic) is similar to + lwaftr-nic, but runs the lwAftr commands in a virtualized environment. [TBC] + +## Jobset parameters + +(The following uses the `lwaftr-nic` jobset as an example.) + +The parameters of each jobset are defined on its [configuration tab] +(https://hydra.snabb.co/jobset/igalia/lwaftr-nic#tabs-configuration). Each +lwAftr branch under test is pointed to a pair of parameters, named `snabbXname` +and `snabbXsrc`, where `X` is an uppercase letter. + +A curve is drawn on each report graph for each `snabbXname/snabbXsrc` pair, +labeled with the `snabbXname` value, and showing benchmarks of the `snabbXsrc` +branch. + +## Reports + +Jobsets list executed jobs on the [Jobs tab] +(https://hydra.snabb.co/jobset/igalia/lwaftr-nic#tabs-jobs). Click on the +[reports.lwaftr](https://hydra.snabb.co/job/igalia/lwaftr-nic/reports.lwaftr) +job, and then on a successful build (indicated by a green mark). In the "Build +products" section, click on "report.html". + +The report has three sections: + +- "Initialization": a summary of the benchmark output data; +- "Line graph": graphs for the various quantities in the output data; +- "Density plot": the distribution of output data for each quantity. + +Each graph shows several curves, one per branch, highlighting the performance +differences among them. diff --git a/src/program/lwaftr/doc/genbook.sh b/src/program/lwaftr/doc/genbook.sh index 421fa9cdf0..d6c3c410cf 100755 --- a/src/program/lwaftr/doc/genbook.sh +++ b/src/program/lwaftr/doc/genbook.sh @@ -31,8 +31,12 @@ $(cat README.configuration.md) $(cat README.benchmarking.md) +$(cat README.continuous-integration.md) + $(cat README.performance.md) +$(cat README.filters-performance.md) + $(cat README.virtualization.md) $(cat README.rfccompliance.md) @@ -43,4 +47,6 @@ $(cat README.counters.md) $(cat README.breaking_changes.md) +$(cat README.ndp.md) + EOF From 24484afc50ef3d7f8dd977b70eef6a1a95b18382 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 7 Nov 2016 17:04:07 +0100 Subject: [PATCH 143/631] Refactor options and data_format config file This whilst refactoring most of data_format is largely to add refactor options so that they are only calculated when the option can actually be set. This also makes some small changes like removing an unused require statement and changing the tab width from 4 to 2 --- .../config/data_format/data_format.lua | 214 ++++++++++-------- 1 file changed, 117 insertions(+), 97 deletions(-) diff --git a/src/program/config/data_format/data_format.lua b/src/program/config/data_format/data_format.lua index 8764ee2b0c..f720b3dbb5 100644 --- a/src/program/config/data_format/data_format.lua +++ b/src/program/config/data_format/data_format.lua @@ -4,127 +4,147 @@ module(..., package.seeall) local lib = require("core.lib") local schema = require("lib.yang.schema") local yang_data = require("lib.yang.data") -local usage = require("program.config.data_format.README_inc") -- Number of spaces a tab should consist of when indenting config. -local tab_spaces = 4 +local tab_spaces = 2 -local function show_usage(status) - print(require("program.config.data_format.README_inc")) - main.exit(status) +local function print_level(level, ...) + io.write(string.rep(" ", level * tab_spaces)) + print(...) end -local function parse_args(args) - local handlers = {} - handlers.h = function() show_usage(0) end - args = lib.dogetopt(args, handlers, "h", {help="h"}) - if #args <= 0 then show_usage(1) end - return unpack(args) +local function union_type(union) + local rtn + for _, t in pairs(union.argument_type.union) do + if rtn then + rtn = rtn .. " | " .. t.argument_string + else + rtn = t.argument_string + end + end + return rtn end -function run(args) - function print_level(level, ...) - io.write(string.rep(" ", level * tab_spaces)) - print(...) - end - function union_type(union) - local rtn - for _, t in pairs(union.argument_type.union) do - if rtn then - rtn = rtn .. " | " .. t.argument_string - else - rtn = t.argument_string - end - end - return rtn +local function comment(opts) + local comments = {} + if opts.mandatory == true then + comments[#comments + 1] = "mandatory" end - function collate_options(name, node) - local options = {} - if node.keys ~= nil then - for _, k in pairs(node.keys) do - if name == k then options.key = true end - end - end - if node.argument_type then - local at = node.argument_type - if at.range then - options.range = at.range.argument_string - end - end - if node.mandatory then - options.mandatory = true - end - return options + if opts.key then + comments[#comments + 1] = "key" end - function comment(options) - local comments = {} - if options.mandatory == true then - comments[#comments + 1] = "mandatory" - end - if options.key then - comments[#comments + 1] = "key" - end - if options.range then - comments[#comments + 1] = "between " .. options.range - end - local rtn = nil - for n, c in pairs(comments) do - if n == 1 then - rtn = "// " .. c - else - rtn = rtn .. " " .. c - end - end - return rtn + if opts.range then + comments[#comments + 1] = "between " .. opts.range end - - function display_leaf(level, keyword, argument, options) - if argument == "union" then argument = union_type(node) end - local comments = comment(options) - local str = keyword .. " ".. argument .. ";" - if comments then - print_level(level, str .. " " .. comments) + local rtn = nil + for n, c in pairs(comments) do + if n == 1 then + rtn = "// " .. c else - print_level(level, str) + rtn = rtn .. " " .. c end end + return rtn +end - local describers = {} - local function describe(level, name, node, options) - local err = "Unknown node type: "..node.type - local options = collate_options(name, node) - assert(describers[node.type], err)(level, name, node, options) +local function display_leaf(level, keyword, argument, opts) + if argument == "union" then argument = union_type(node) end + local comments + if opts then comments = comment(opts) end + local str = keyword .. " ".. argument .. ";" + if comments then + print_level(level, str .. " " .. comments) + else + print_level(level, str) end - function describe_members(node, level) - if level == nil then level = 0 end - for name, n in pairs(node.members) do - local options = collate_options(name, node) - describe(level, name, n, options) +end + +local function show_usage(status) + print(require("program.config.data_format.README_inc")) + main.exit(status) +end + +-- Contains verious option handling code. +local options = {} + +function options.key(keys) + return function (name) + for _, k in pairs(keys) do + if name == k then return true end end + return false end - function describers.scalar(level, name, node, options) - display_leaf(level, name, node.argument_type.argument_string, options) +end + +function options.mandatory(name, node) + if node.mandatory then return node.mandatory end +end + +function options.range(name, node) + if node.argument_type.range then + return node.argument_type.range.argument_string end - function describers.table(level, name, node, options) - if node.keys then - print_level( - level, - "// List: this nested structure is repeated with (a) unique key(s)" - ) - end - print_level(level, name.." {") - describe_members(node, level + 1) - print_level(level, "}") + return nil +end + +-- Contains the handlers which know how to describe certain data node types. +local describers = {} + +local function describe(level, name, node, ...) + local err = "Unknown node type: "..node.type + assert(describers[node.type], err)(level, name, node, ...) +end + +local function describe_members(node, level, ...) + if level == nil then level = 0 end + for name, n in pairs(node.members) do + describe(level, name, n, ...) end - describers.struct = describers.table - function describers.array(level, name, node, options) +end + +function describers.scalar(level, name, node, is_key) + local opts = {} + if is_key then opts.key = is_key(name, node) end + opts.mandatory = options.mandatory(name, node) + opts.range = options.range(name, node) + display_leaf(level, name, node.argument_type.argument_string, opts) +end + +function describers.table(level, name, node) + if node.keys then print_level( level, - "// Array: made by repeating the keyword followed by each element" + "// List: this nested structure is repeated with (a) unique key(s)" ) - display_leaf(level, name, node.element_type.argument_string, options) end + print_level(level, name.." {") + describe_members(node, level + 1, options.key(node.keys)) + print_level(level, "}") +end + +function describers.struct(level, name, node) + print_level(level, name.." {") + describe_members(node, level + 1) + print_level(level, "}") +end +function describers.array(level, name, node) + print_level( + level, + "// Array: made by repeating the keyword followed by each element" + ) + display_leaf(level, name, node.element_type.argument_string) +end + +local function parse_args(args) + local handlers = {} + handlers.h = function() show_usage(0) end + args = lib.dogetopt(args, handlers, "h", {help="h"}) + if #args <= 0 then show_usage(1) end + return unpack(args) +end + +function run(args) local yang_module = parse_args(args) -- Fetch and parse the schema module. From 6fb48f8c57b0ed6f0fb4aef8a8e0d35a8d4bc826 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 7 Nov 2016 18:10:48 +0100 Subject: [PATCH 144/631] Fix some small code cosmeic issues --- .../config/data_format/data_format.lua | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/program/config/data_format/data_format.lua b/src/program/config/data_format/data_format.lua index f720b3dbb5..3042016a46 100644 --- a/src/program/config/data_format/data_format.lua +++ b/src/program/config/data_format/data_format.lua @@ -76,10 +76,6 @@ function options.key(keys) end end -function options.mandatory(name, node) - if node.mandatory then return node.mandatory end -end - function options.range(name, node) if node.argument_type.range then return node.argument_type.range.argument_string @@ -105,18 +101,13 @@ end function describers.scalar(level, name, node, is_key) local opts = {} if is_key then opts.key = is_key(name, node) end - opts.mandatory = options.mandatory(name, node) + opts.mandatory = node.mandatory opts.range = options.range(name, node) display_leaf(level, name, node.argument_type.argument_string, opts) end function describers.table(level, name, node) - if node.keys then - print_level( - level, - "// List: this nested structure is repeated with (a) unique key(s)" - ) - end + print_level(level, "// List, key(s) must be unique.") print_level(level, name.." {") describe_members(node, level + 1, options.key(node.keys)) print_level(level, "}") @@ -129,10 +120,7 @@ function describers.struct(level, name, node) end function describers.array(level, name, node) - print_level( - level, - "// Array: made by repeating the keyword followed by each element" - ) + print_level(level, "// Array, multiple elements by repeating the statement.") display_leaf(level, name, node.element_type.argument_string) end @@ -140,7 +128,7 @@ local function parse_args(args) local handlers = {} handlers.h = function() show_usage(0) end args = lib.dogetopt(args, handlers, "h", {help="h"}) - if #args <= 0 then show_usage(1) end + if #args ~= 0 then show_usage(1) end return unpack(args) end From 2a5441649c2749095f1ea5fb4d81423caf00879d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 13:43:51 +0100 Subject: [PATCH 145/631] ctable: hash_fn param is optional If no hash_fn is given, ctable will provide one. --- src/lib/README.ctable.md | 10 +++++++++- src/lib/ctable.lua | 25 +++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md index f05ebaa774..c674e70a33 100644 --- a/src/lib/README.ctable.md +++ b/src/lib/README.ctable.md @@ -62,7 +62,6 @@ following keys are required: * `key_type`: An FFI type (LuaJIT "ctype") for keys in this table. * `value_type`: An FFI type (LuaJT "ctype") for values in this table. - * `hash_fn`: A function that takes a key and returns a hash value. Hash values are unsigned 32-bit integers in the range `[0, 0xFFFFFFFF)`. That is to say, `0xFFFFFFFF` is the only unsigned 32-bit @@ -71,6 +70,9 @@ hash value in the correct range. Optional entries that may be present in the *parameters* table include: + * `hash_fn`: A function that takes a key and returns a hash value. + If not given, defaults to the result of calling `compute_hash_fn` + on the key type. * `initial_size`: The initial size of the hash table, including free space. Defaults to 8 slots. * `max_occupancy_rate`: The maximum ratio of `occupancy/size`, where @@ -258,3 +260,9 @@ Hash the first 48 bits of a byte sequence. — Function **ctable.hashv_64** *ptr* Hash the first 64 bits of a byte sequence. + +— Function **ctable.compute_hash_fn** *ctype* + +Return a `hashv_`-like hash function over the bytes in instances of +*ctype*. Note that the same reservations apply as for `hash_32` +above. diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 7903e0066b..4acf07eac0 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -94,8 +94,9 @@ end -- FIXME: For now the value_type option is required, but in the future -- we should allow for a nil value type to create a set instead of a -- map. -local required_params = set('key_type', 'value_type', 'hash_fn') +local required_params = set('key_type', 'value_type') local optional_params = { + hash_fn = false, initial_size = 8, max_occupancy_rate = 0.9, min_occupancy_rate = 0.0 @@ -106,7 +107,7 @@ function new(params) local params = parse_params(params, required_params, optional_params) ctab.entry_type = make_entry_type(params.key_type, params.value_type) ctab.type = make_entries_type(ctab.entry_type) - ctab.hash_fn = params.hash_fn + ctab.hash_fn = params.hash_fn or compute_hash_fn(params.key_type) ctab.equal_fn = make_equal_fn(params.key_type) ctab.size = 0 ctab.occupancy = 0 @@ -491,6 +492,7 @@ function hash_32(i32) return uint32_cast[0] end +local cast = ffi.cast function hashv_32(key) return hash_32(cast(uint32_ptr_t, key)[0]) end @@ -510,6 +512,22 @@ function hashv_64(key) return hash_32(bxor(hi, hash_32(lo))) end +local hash_fns_by_size = { [4]=hashv_32, [8]=hashv_64 } +function compute_hash_fn(ctype) + local size = ffi.sizeof(ctype) + if not hash_fns_by_size[size] then + hash_fns_by_size[size] = function(key) + local h = 0 + local words = cast(uint32_ptr_t, key) + local bytes = cast('uint8_t*', key) + for i=0,size/4 do h = hash_32(bxor(h, words[i])) end + for i=1,size%4 do h = hash_32(bxor(h, bytes[size-i])) end + return h + end + end + return hash_fns_by_size[size] +end + function selftest() print("selftest: ctable") @@ -575,8 +593,11 @@ function selftest() local function check_bytes_equal(type, a, b) local equal_fn = make_equal_fn(type) + local hash_fn = compute_hash_fn(type) assert(equal_fn(ffi.new(type, a), ffi.new(type, a))) assert(not equal_fn(ffi.new(type, a), ffi.new(type, b))) + assert(hash_fn(ffi.new(type, a)) == hash_fn(ffi.new(type, a))) + assert(hash_fn(ffi.new(type, a)) ~= hash_fn(ffi.new(type, b))) end check_bytes_equal(ffi.typeof('uint16_t[1]'), {1}, {2}) -- 2 byte check_bytes_equal(ffi.typeof('uint32_t[1]'), {1}, {2}) -- 4 byte From ee29d11186a80913a375f5f68f4b40c431dc4c8e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 2 Nov 2016 14:43:59 +0100 Subject: [PATCH 146/631] Ctable supports load/save This allows a ctable to be persisted to disk or to a binary stream and cheaply loaded back again at run-time. --- src/lib/README.ctable.md | 16 ++++++ src/lib/ctable.lua | 102 ++++++++++++++++++++++++++++++++++----- 2 files changed, 106 insertions(+), 12 deletions(-) diff --git a/src/lib/README.ctable.md b/src/lib/README.ctable.md index c674e70a33..3651c1cca4 100644 --- a/src/lib/README.ctable.md +++ b/src/lib/README.ctable.md @@ -83,6 +83,15 @@ Optional entries that may be present in the *parameters* table include: * `min_occupancy_rate`: Minimum ratio of `occupancy/size`. Removing an entry from an "empty" table will shrink the table. +— Function **ctable.load** *stream* *parameters* + +Load a ctable that was previously saved out to a binary format. +*parameters* are as for `ctable.new`. *stream* should be an object +that has a **:read_ptr**(*ctype*) method, which returns a pointer to +an embedded instances of *ctype* in the stream, advancing the stream +over the object; and **:read_array**(*ctype*, *count*) which is the +same but reading *count* instances of *ctype* instead of just one. + #### Methods Users interact with a ctable through methods. In these method @@ -156,6 +165,13 @@ Return true if we actually do find a value and remove it. Otherwise if no entry is found in the table and *missing_allowed* is true, then return false. Otherwise raise an error. +— Method **:save** *stream* + +Save a ctable to a byte sink. *stream* should be an object that has a +**:write_ptr**(*ctype*) method, which writes an instance of a struct +type out to a stream, and **:write_array**(*ctype*, *count*) which is +the same but writing *count* instances of *ctype* instead of just one. + — Method **:selfcheck** Run an expensive internal diagnostic to verify that the table's internal diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 4acf07eac0..d31c060f73 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -171,6 +171,45 @@ function CTable:get_backing_size() return self.byte_size end +local header_t = ffi.typeof[[ +struct { + uint32_t size; + uint32_t occupancy; + uint32_t max_displacement; + double max_occupancy_rate; + double min_occupancy_rate; +} +]] + +function load(stream, params) + local header = stream:read_ptr(header_t) + local params_copy = {} + for k,v in pairs(params) do params_copy[k] = v end + params_copy.initial_size = header.size + params_copy.min_occupancy_rate = header.min_occupancy_rate + params_copy.max_occupancy_rate = header.max_occupancy_rate + local ctab = new(params_copy) + ctab.occupancy = header.occupancy + ctab.max_displacement = header.max_displacement + local entry_count = ctab.size + ctab.max_displacement + 1 + + -- Slurp the entries directly into the ctable's backing store. + -- This ensures that the ctable is in hugepages. + C.memcpy(ctab.entries, + stream:read_array(ctab.entry_type, entry_count), + ffi.sizeof(ctab.entry_type) * entry_count) + + return ctab +end + +function CTable:save(stream) + stream:write_ptr(header_t(self.size, self.occupancy, self.max_displacement, + self.max_occupancy_rate, self.min_occupancy_rate)) + stream:write_array(self.entries, + self.entry_type, + self.size + self.max_displacement + 1) +end + function CTable:insert(hash, key, value, updates_allowed) if self.occupancy + 1 > self.occupancy_hi then self:resize(self.size * 2) @@ -550,21 +589,60 @@ function selftest() ctab:add(i, v) end - -- In this case we know max_displacement is 8. Assert here so that - -- we can detect any future deviation or regression. - assert(ctab.max_displacement == 8) + for i=1,2 do + -- In this case we know max_displacement is 8. Assert here so that + -- we can detect any future deviation or regression. + assert(ctab.max_displacement == 8) - ctab:selfcheck() + ctab:selfcheck() - for i = 1, occupancy do - local value = ctab:lookup_ptr(i).value[0] - assert(value == bnot(i)) - end - ctab:selfcheck() + for i = 1, occupancy do + local value = ctab:lookup_ptr(i).value[0] + assert(value == bnot(i)) + end + ctab:selfcheck() + + local iterated = 0 + for entry in ctab:iterate() do iterated = iterated + 1 end + assert(iterated == occupancy) - local iterated = 0 - for entry in ctab:iterate() do iterated = iterated + 1 end - assert(iterated == occupancy) + -- Save the table out to disk, reload it, and run the same + -- checks. + local tmp = os.tmpname() + do + local file = io.open(tmp, 'wb') + local function write(ptr, size) + + file:write(ffi.string(ptr, size)) + end + local stream = {} + function stream:write_ptr(ptr) + write(ptr, ffi.sizeof(ptr)) + end + function stream:write_array(ptr, type, count) + write(ptr, ffi.sizeof(type) * count) + end + ctab:save(stream) + file:close() + end + do + local file = io.open(tmp, 'rb') + local function read(size) + return ffi.new('uint8_t[?]', size, file:read(size)) + end + local stream = {} + function stream:read_ptr(type) + return ffi.cast(ffi.typeof('$*', type), read(ffi.sizeof(type))) + end + function stream:read_array(type, count) + return ffi.cast(ffi.typeof('$*', type), + read(ffi.sizeof(type) * count)) + end + ctab = load(stream, params) + file:close() + end + os.remove(tmp) + end -- OK, all looking good with the normal interfaces; let's check out -- streaming lookup. From 6d93fedc7c5f47fe95370c2eac5af1011a4282bf Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 8 Nov 2016 09:47:43 +0100 Subject: [PATCH 147/631] lwaftr binding table uses ctable instead of podhashmap Also use the automatically provided hash function. We'll see how this goes! --- src/apps/lwaftr/binding_table.lua | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index f51e608fcc..fa37f00107 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -69,7 +69,7 @@ local stream = require("apps.lwaftr.stream") local lwdebug = require("apps.lwaftr.lwdebug") local Parser = require("apps.lwaftr.conf_parser").Parser local rangemap = require("apps.lwaftr.rangemap") -local phm = require("apps.lwaftr.podhashmap") +local ctable = require("lib.ctable") local band, bor, bxor, lshift, rshift = bit.band, bit.bor, bit.bxor, bit.lshift, bit.rshift @@ -140,15 +140,6 @@ function is_fresh(stream, mtime_sec, mtime_nsec) return res end -local hash_i32 = phm.hash_i32 -local function hash_softwire(key) - local ipv4, psid = key.ipv4, key.psid - -- PSID is only 16 bits. Duplicate the bits into the upper half so - -- that the hash function isn't spreading around needless zeroes. - psid = bor(psid, lshift(psid, 16)) - return hash_i32(bxor(ipv4, hash_i32(psid))) -end - BTLookupQueue = {} @@ -224,8 +215,8 @@ function BindingTable:lookup(ipv4, port) local psid = self:lookup_psid(ipv4, port) lookup_key.ipv4 = ipv4 lookup_key.psid = psid - local res = self.softwires:lookup(lookup_key) - if res then return self.softwires:val_at(res) end + local entry = self.softwires:lookup_ptr(lookup_key) + if entry then return entry.value end return nil end @@ -373,8 +364,8 @@ local function load_compiled(stream) local psid_map = rangemap.load(stream, psid_map_value_t) local br_address_count = stream:read_ptr(br_addresses_header_t).count local br_addresses = stream:read_array(br_address_t, br_address_count) - local softwires = phm.load(stream, softwire_key_t, softwire_value_t, - hash_softwire) + local softwires = ctable.load( + stream, { key_type = softwire_key_t, value_type = softwire_value_t }) return BindingTable.new(psid_map, br_addresses, br_address_count, softwires) end @@ -465,8 +456,8 @@ local function parse_softwires(parser, psid_map, br_address_count) end } - local map = phm.PodHashMap.new(softwire_key_t, softwire_value_t, - hash_softwire) + local map = ctable.new( + { key_type = softwire_key_t, value_type = softwire_value_t }) local key, value = softwire_key_t(), softwire_value_t() parser:skip_whitespace() parser:consume_token('[%a_]', 'softwires') From f90748941bb1e146746873b27a78b16c9895ac2a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 8 Nov 2016 09:52:39 +0100 Subject: [PATCH 148/631] Bump binding table version Switching to ctable slightly changes the binary format for the compiled binding table so we need to regenerate them. --- src/apps/lwaftr/binding_table.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index fa37f00107..5d86f6a9ce 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -74,7 +74,7 @@ local ctable = require("lib.ctable") local band, bor, bxor, lshift, rshift = bit.band, bit.bor, bit.bxor, bit.lshift, bit.rshift local BINDING_TABLE_MAGIC = "\0bindingtabl" -local BINDING_TABLE_VERSION = 0x00002000 +local BINDING_TABLE_VERSION = 0x00003000 local binding_table_header_t = ffi.typeof[[ struct { uint8_t magic[12]; From 803f10e85ef789b86e8bab9135b7f6675af8f4c4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 8 Nov 2016 11:38:44 +0100 Subject: [PATCH 149/631] Grouping has no type. --- src/lib/yang/snabb-softwire-v1.yang | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index ddde2a1d64..238ebee337 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -20,7 +20,6 @@ module snabb-softwire-v1 { "Configuration for Snabb lwaftr."; grouping traffic-filters { - type string; description "Ingress and egress filters describing the set of packets that should be allowed to pass, as pflang filters. pflang From 9c0739605e424222f91c7574a303d8b566578918 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Tue, 8 Nov 2016 11:50:21 +0100 Subject: [PATCH 150/631] Let out-ipv6-* counters also count internally generated ICMPv6 packets, and update docs (#547) --- src/apps/lwaftr/lwaftr.lua | 2 ++ src/apps/lwaftr/lwcounter.lua | 2 +- src/program/lwaftr/doc/README.counters.md | 1 - .../doc/images/decaps-queue-to-internet.dia | Bin 4739 -> 4740 bytes .../doc/images/decaps-queue-to-internet.png | Bin 57210 -> 56949 bytes .../counters/in-1p-ipv6-out-1p-icmpv6-1.lua | 2 ++ .../counters/in-1p-ipv6-out-1p-icmpv6-2.lua | 2 ++ 7 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 4986149b80..9be7acd67e 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -180,6 +180,8 @@ local function init_transmit_icmpv6_reply(lwstate) num_packets = num_packets + 1 counter.add(lwstate.counters["out-icmpv6-bytes"], pkt.length) counter.add(lwstate.counters["out-icmpv6-packets"]) + counter.add(lwstate.counters["out-ipv6-bytes"], pkt.length) + counter.add(lwstate.counters["out-ipv6-packets"]) return transmit(o, pkt) else counter.add(lwstate.counters["drop-over-rate-limit-icmpv6-bytes"], pkt.length) diff --git a/src/apps/lwaftr/lwcounter.lua b/src/apps/lwaftr/lwcounter.lua index a6b3734320..149aa6ce0d 100644 --- a/src/apps/lwaftr/lwcounter.lua +++ b/src/apps/lwaftr/lwcounter.lua @@ -22,7 +22,7 @@ counter_names = { "in-ipv6-bytes", "in-ipv6-packets", --- Outgoing traffic, not including internally generated ICMP error packets. +-- Outgoing traffic, including internally generated ICMP error packets. "out-ipv4-bytes", "out-ipv4-packets", "out-ipv6-bytes", diff --git a/src/program/lwaftr/doc/README.counters.md b/src/program/lwaftr/doc/README.counters.md index 71b51e564e..3f3b0382a5 100644 --- a/src/program/lwaftr/doc/README.counters.md +++ b/src/program/lwaftr/doc/README.counters.md @@ -44,7 +44,6 @@ Counters: - **drop-in-by-policy-icmpv6**: incoming ICMPv6 packets dropped because of current policy - **out-icmpv4**: internally generated ICMPv4 error packets -- **out-ipv4**: all valid outgoing IPv4 packets - **drop-too-big-type-but-not-code-icmpv6**: the packet's ICMP type was "Packet too big", but its ICMP code was not an acceptable one for this type - **drop-over-time-but-not-hop-limit-icmpv6**: the packet's time limit was diff --git a/src/program/lwaftr/doc/images/decaps-queue-to-internet.dia b/src/program/lwaftr/doc/images/decaps-queue-to-internet.dia index 3fd7b72d3d214f69b6740283acd73e3155c30fc4..ab33d20f41bba339737b0198b48e1c54d509952c 100644 GIT binary patch delta 4581 zcmVTYzW zkbDtj^C+px(Q^|Q6Mvsa7=|eR8a{XWa_GFbGPYG?iuCo+t~#blahNp*jB^GsHN#gXcBy<&*y zF!W;>%^&mKWAk=n->2Cu%=X>5?-|NvUqWyyF_Rdz6S#UzpC>g8@_-$xi5R`NhLctP z5{G38VX|IqzU_8He;2E0k_EG9z1lrz`{t+kVWG6GwHS5PVxw8Ts<9UEuE!V|4eno$ zyFAq)imW^astl^^RF!dORW`eq>{FGCEm;(*jO~_;Sns;r;js?UW!a-93(({rP?Jel zP3Ao-GXJ2*M!g2nV-9KzYJ8e%{EzVW{DU5&jULmkdMrG_f5ybaSYXnlI$JT=lS~P6 zw~b*!uJveL;9J*~yyZK5<)8Sdb~!xLB(D=>em;5#7E%17?g?Tg$KKgu`-$4aLYIU6|BK3JlR{?jdo-ZJYs-HfJcBwfJZ~)(cd)TtFd!< zXFM8H&nNoTtF5D?QdloJ$rvGb((^GUTua7!VYH^HNiqqOnSGXqPguQ3nu~Bt<4b?% z(`ES%>f(Nl8UB8|1#RpW#<1cH@s_ixCvp;GS^B(te?ep1ap*}LCiBl92dmPntH`|* z*ixcdeR`2=U0hYR#n<1A9<}yvwv{|J%1W+B@4wbFT3L}bA4Z!GQ@s6QzZ&)|-I5!| zBf27t>Y`c);fvD9!kJ)qMifGV0m%W9Q(1PvZwBQzU4XKTb#frAlU=dScy#;4IyUB> z#8QKufAoYf@-RmD3=r=jUauY}vv3+LSL--1-`x>GP*zd!=35%i27y9s*eF55r-&%w ztG=PotP(^KfMIW7h}#)^CPuK_61)Xi{}HB(P>(a!i1lmwoQ!_4484k5YF|cO6}f6m zAdig^_E`HY$O?efLzwiaV3HeQRNX0$VT$M&e-V%AMcQ7vo~ocKy9{TuXqvZ|Q6FaO zV=imsZ7wrn>!WT?bI|K7*oHQ2E13q5)>++J_zb6-W_QQ&L4vlcFj^$3?+hr`g%oW< zYPZ=dM-}(tuDgu+I9g5~(=7V8-nWCe-{gNQOe3k;`g<0&THl*egRrZfRd8*lD`(LSW2f_7>b}#R3!hB2Icb28gon-e@esnnD?QHkAxsO`H z+nn;Y5nZLdJC>-d5Z^Z9>d5bb^tX+;T@<(i#;Ad;9YnZsu}hf@f4)QPD|M;y5A634 zsBH%UsS2zaxQ2z_mS}h@I7QuqFeU}8f6s8R5kPz*O|F?q|JMI-#m^ubY@l-zEuZ*g zsX0)XPad*#q505kvPvKF=O_!oi!h8A!LA{WIE0K(%!|-4WQ=h4hz>$Mf|c|n7MzZe z?aV?eDQirIHXShDdRL0T3d#`3-j+0NGK>MjX_#e_<@h zSet7-CzV)9*h<2Z5L2Hjh<>k$6p?KuA6(kj{UHxYsP}uVaYGlFQors08akui(FWZ) z=^gp~=UmvFE~9QfqROftCW4yxl=Pj_ReiqQO4=#C?F{#8CjocSBr^$-oyNGE&1#pP zuW64jF4_^c|De;|A3vy~^TyGQ|@1v=X!ouxbTdG^CgnG3U_;DtNb zhLPXz9;{Wjz!$RtYe2D9vj&&}c3|XxzWug-c0d(Cl{Q9lBho@x&!lBnu%L53HDY~7 z3-s_x`PhiTQr#ixA&#w7(bn-dWXK@Tf@HOb@<}vZET1^U2Zo6c6zh^Be*^)7GYJsv zZ~{sPU2rs9sJ<}@QzL{H4KxH}hWfDH8{u?$GVu9<&u@pZp*kxa>DSmmxUe>qGU|r8 z(Ka9TZDx>tW`Z$CTMl6Z!!6o^-^NU1hstC+Ulb50&q(B~iiDCf^+ioA7>F z`R)!T6B6Db;T;m*A>qA`e}s38oa`84n2dJ~rf(_ldbfa_cgT5fkn>)wQb>Bgz@&Fg ze2xV6ubB zepx0vb!4*3AuM+05yr8WMSLQd*(*3DQorYlpO@MG-+{h7juufq(qH`aF`WGjR{T+{ zc+>cRgB3rp6_1C+IOYc9JXVg9_5yOk!%|9x1~9^+A=#t93p?i4a6T`iy>*a~Elo#K&^z#2={e5O+_&mpvnk z_*i*}jZ$DV6tc*Duwq*i2*4J=mO&${VxS5_+`Uu)k43-5e>q0L#uaW#!t}e6BmDGh z2bWJ2DhCIu95lI^AB7C@Gse^0#Rj%=ho*{9#)boyISFsc#byJlv=JWXnhCxV>>r?6HXHbivFq=*xu zWl>vp&Cpou^`#`AcAmvkNkD~~QAj{N%>>kLkNN}WpAt8f3S`Xt-z3cbz!diF=^hjd zb4Dojg~I2Qz3}RJyNZ&@C$2Y0WauxE!K@?YQzkDrf04oeQ4<;Jdwr44t;UTJDLlb? z?VwDR6++DsaLi}6r*W@+zPdftA#J`;k4?T>w-AFiUrin44XPr)huMSq&ENmE4zjS0 za=1wcN2X+vb1wQP9;UF`NMl;s9U|IDvE`AJkPER;9n9s)6iIqM-+ECnTES?As>#c; zTHWN{f8#;wiI7KxttTSVr}z$GkK;||JyQIfY}aq=D4t#Y^-ur#{fS>?;WB;!`}GL+ zE7og?E4=OZfJFS0uwT1%J}NgbMcirYBPYF0h;V_ua)8z*qG)OI-rB^!z}6-bI-3AT z2aXOL{p2{hbh1x)rE>^++AFxHl&~IG?n`ple_8DCfTM@u zsDd^sKR_E5>g1FYHB1*3X`869w=OEmHc?^FMFl|ZP*8&=Di@o3^v`Ji&tP7v<9Wlm zf9`cW%;b^x+eifo!N>H~U&@L^{0?UEuOZxnmIQc&mwg-$2aW}jj=p(x_{m<<+=GWlcvhj0}0e4-G& zlyFqX$%K#;grp!O1tF&H`BUoAoDz*+(Y8rZ+SGRp|pdQE9 zKs`?3guIJ(pHf#LAvy>N)*xm*sHR&;0ct`OpoY&4)WCy!w6GBM5a8Nn;~H~A)v(8y z>{6$7uyIcfo@sf6+X{8xV*+9QUa(T7@u}n5Ust1hI$CK1Z)zVdf}@d+!``sRf5k;t zsb74gXpfik02p+1$cRVBLmla^q|maEUz`tzd<}x+yfP`%b$;_{UtB=FQICfwnWajh zw(0azClBLxA44~PT4zt;Z1iW%B}U)vm#iT#tHU|r2j#l5)=*5T}vaMt-MqU znf6-B+ho}5_u@iGwWApYUJbk&cs20qqw#9yq*x0vPq85yMq4>-Q-5j>e{+Ln9u9|(07?~o(B!%Uj4<6@Y8wbWz2mFFYdv6dI_Me+VZ zIJK)u1yT*98b~#e>f@1WH&tvNH&{48YdB^r$lZ z8Ppy2O^KN>%`nl$f0XHToW)aVUW{Nzf#$`qqg+(;V&LXbLLKJ5wSnW@xn-X zN|{C?%KD=XZ#Er5FN8REokXC`K%0R!18weuHgn`=oA7*2fA|pRCclbOue=%QT>~Mr zUl`(izPuTtmPg>t@2d%T^Lg{;rnrYYpEpX(0o=UD14or7xoM1qm`4S+Ejkb(qP5Tx zb{&Rqhojqec1Ac2maBCf*mDy9wGP)>LORm37izwo@5H*#mgRvs*AeBK-a#$GRfCwE znis|bBgwH)f8B&uv8{f2GEb^LBePKx-;!7nptC+HN-})x%g5jj2Fb!^36U zo$h(Kjqjv5kQHGav;7WG9Bt!c%mv8 z{1;~PJE;TY9%5LSa&Ld2t%GADk9d-p)*9U7vXGCle>$%tq%0iNZO`PYXgkdjGWvi&jO}4Bgeo@Hf40e!yA(6=zvj zAA>SY>OR_U&V3A~ue9xiXU-b+(O7t*rFX&1>m*EO_Vyb+RNWx7ji_o0{DbXgYv zZWgy2f3%j%He#}$NvmD$`_?+e+iRUyCbZ_mX!Bu0ONvsy(aJ5gGi5)P6Z{$li&Qh% z8bacxjEWl5Yl!xdR5Ek?ppr~-+dfj8u~vz+HG3hso*n1%R6^Dm%vUg9!F;_e^VLzc z*$_#d%o9>to3&`N&OM9}HulH4wOLP6Fj$X}f7i*)T7jdA&bXrX06>$D+Nd)VN*SVf zmU-Bh#*ISEsqS|M=9Gmsfzf%v+Do_sTsz945oWP+iU8Rz8QGLGWW$4;TC8n*tbq-C zgcWp+35{#rWDgH&e<_Z{N%Wha0o{&-ZlVVfOW^c%%Q!vtaS{ P3+n#?IPK9317raJg3chzDSWrJpt;Gg)BmQn|Fge&!YQv9*&Y=5q`P852nA(vvi%z zu8XTS*GKFmpa|G(ih-#qN! z^B|c8*=%$(`j-BF{Z1e>eTZgXuK&y5o~-iY>A^G*R%UCNzAY*0f3YzxKYkTFb`5@i zUmiQtcONA4ID8$SDmF%*QtS>PkNIpYJ>O^TM22y=m`u|wsqXJ?p2-TZI8uGCR}2vy zhJFmA`D4C&Y~F6{`!t({*}fb1Jwv(dj}V+n%p^wb1g;*_=SdBNJYa`vB1Z46;bfJ+ z#9>)Nn5-9@Z@b;lf5j@AWWg+2uXfMbzWFKMER?pj7Nf3OY&5G^HP!;&^%x_g!TsxT zm!~>Jk(I|ll|hxAsxt1Z%4YYHeX4S?C5u9pvE7mp>s^;SJk}w)EPK>s0h;^+YBK4n z$-HMp<{$LfsMjER%t4JojZagJ{}KM4f6!yJ(PP?GkA)}Lf0%d}3ru=cXDbGKk|{y% zHZlp)FWSk>&1~#(3YD(ewH~bteCxWBw|s}M{1YG5E{A8Df=7y1q{hf1k+%r#SqCwoh~(T)s)M-1=?@Cfh-@Mvf}`kN+vHFgf~ zj7MYY`9%NpYU?Pe6xK^lGDgUq^n8p7*OIYb7_Dh)l1##6W}l_u6IL&h<|5qE_|o6` zbXmTGy0~9shQHr#K^wb;F|0U4yya}_iJSykmOk%Zf6y3r9C{Lm$^7%j!K(DrRpj0Y zY$?&KKE24bF0Lxu;_Gilk6L>-+e)4qWhK|6_h0K7t*pqJ52MY8Dc=6DUk!VfZpjVf z5nT~Rby2N@@I`54;Y_eQBMKqGfaCzlsVqC-H-qwT zRo_r(RtcgAz_2$k#O;hd6C+q|3El#%{|M7XsK=RV#QHUTPDa02hF--jwJ)Qtid;1& zkjKUdd#rsHWCg(LAxwHyFv*QDs_vA>Fhz8Xe~8EQB5f~SPgPKrU52w+G|k(~s1LLC zF_*RRHkTQ(^-(vcIp}p3Y(pEil}v+2>#S}qe1=m^v%BN?AVJ$z7%h_2cLo&eLW(va zwcG5Kql$ZR*ImYZ94#l0X%_uk@7qD#Z}PttrjgWa{XL6Xt?x~#LD*H#s=G)#nu~lF zfBS{9kUhX|!4b2f^j1Nn{jXHGgW!5byO;MiVZJ5qJIm7LPO|$cKRTT7cDDQ5+(#|p zZBBXHh_2G!9ZOVJh;JKlb>#Oz`rAg_E(%-$W7I&_4kFyR*riN{Ki?ttmAcgU2ljgi z)V70wR0Y-yT*Ja|OEkO{oTBbQ7?T3le`h$@2p~R@CfCfQf9rp^;%5*IHqbeVmQQ@L z)Ep?xCl6V=(0ph%S)~v8bCiYPMHt45VAl{w974t?=0#{2GDf(2LV#+*iNV~__2i?<)G_V?JMQb~vuL!`LB00uO8QResy^RtCGC{nc82@4lYqNul9`0aPGj87X0=OC zSa%{TJ`nN*g1&5osZ;XVS7OSkO728nM2k z1%(6~F2Kk`XA~{SvmjY5qI?of7ba%#9OaLbEXc!29O<_qR4`1aV6$W(e^zidv4S1` zKbrx09=@-Yp>g9n#+$q`w!d6!PCMF#lZ> z-yEgvbU;DJ?x){RNOz3h}bO<7JpQN-Aw)y+^?! zwFjtFh!}BWx{p==ER1%S;|-f3oA=O?C_>JDBWX zvR{_TP92%-atMo^d4zGSWf7kUX7&nBiPW#T;^$?y|97A-w|NsI{l!lo!`aVZ#UI6r zH$4zISn&f}@pwp#V{S0cW92AmFCZs8ETuHLrv=CfPauSC?y2JBoIox~0n?6wX?CG{ zsFETEsCG%H#$5$lfBa#BEy9;pSkWp&qns*Gjh`8+y@V@(wc`X_{F=O10JW0@H8;Tz z;UGs=u%0vav={|a z$09^*lmernkVWo;72Bmi0JZ?O3>sM#0~HhE?xg~FEcz|Ze=!0!u5eQlreBpD;iq3q zxO}2eIXF<|pjj?8&Tgn2JhLu?#COn2Ob}#`@gY9L{k9SlSd$6jNrJFFj@y_=RC5@n z`sOgZp0d`EVq*$S2AE8bRubvZsfF|7SV^RF17=&aMEgxZrRJErjRQu$BTWB2pvn(d zXGj6E0J8LafA}SLWFzg#KK&M_)8C4MQH>biH8Tt5X;Nc65$q&5g|+IiA)-?zMVt^V zi`uelhQ?a2FD3c3^DLf90xHyuLIUb(CZKkE)E_wil(?x>AYSQn#fS!>x*>mI&O?e;R)7j zGi9o*5Nb|?V?ML(jeG6$)$OScY4e48Z1UB*g&4H?YU&_wP!;(-%pS~d{{F9ZkcD-W z!%aFkG9`n!GHl z)lKd_e;%Zs2zf-BG_9W6=mC~FzBNKIC>b4 zDrlqf1GG`0PEI*d!*o%RwuuUR>!PA;6BPzsQ~=Zt1vO}*aT^x*H(2MXUf22dMzPl`1%*yf=yWn|_9=D~iXxtm*T!Gx z)Z-LR$h&CwDRmVRqJxlN4Pw@VYPy9Kpe9rSYWUnh4Lqnv3ky*X0j^y(t}!=M4SS5q zE_GT58~4=UnU+Vmtx)$pCJ@%I1uIn=pE|Dnbv3%Dqm?%BruN|?I2!3V>}!aS*jFj zn@%rv@-S}qF?92%b@mj_Mt{~^V)Wg9$r|#qI-C=Za1L49QVarhRNJk@fSmacyO34@ zE~fCX7>?h$CEorhPA3@<$8jMDf8sb0#~GY9JE=X=$SxETW988yP8~H3P>p-k9w(tR z0QD*3Mw(NPjSUz#!&qv_TZ)=vpsmf>UA#KN^l_S{GD@U&&LD3N1wKVeC%1fn? zX|JWcO@_UGEiQyqJDO48)xfKPR|Bs;8n0$finS2)6dR&pw3Wj)^{3`Ae@AJm8P1JR zKL$~@j1q83AB$2K%heR3UNWKfflycR4mrX*%%tf$E{5q>OFiaWc|NioYkBcn6t6FY zQ@ff}Ak{#sfm8#jJ|3xdQ^n?SgM|Y`J~bUOMQel$U)~w9ni@%bUZ=3kvUGWqq&G#D z+s)M~eaN4qEHoq={MJWlf0|{MFBj^P^W88|>?-;ZH|Qtq8GA#NWtjpj<4^E4@O8r5 zld9WwFoD3Nfk^|C1}1$xChe#gj+g)>(rjdidbo`+Dx@@L3rT8TNrA}O@oAI!asP6& zOyg+!0y!W@tdQHRDWMPnhxHG|rkOJ~Z7^vNI&B)_6@1zpjdM|if4b`&0;2{-4U8HX z_3;?BtNOzMDfPQF%c(K;e0IK^T7M-sS&)Y}akPjav-+r&i9oA=R9fw(JdqE`s}X-A z*7Bl=wVPQ4W(~|5m^Co#o|yHwIGz4B`o%PG*89wNz3m3`G^yKJTvaNvVH$i$k1Erj zLET~Bl$Z(A3=>^Uf0<6lSv-~I#Rzs3XkH9E%0)FVX0D1mndEd#huA8tBn_y_bi~63 zGl=O)OMP)(*@Af%B&$V~o8vVnP=s@&y7tX#QvOBv#Pfgfo_Gf{XYQmH5&MH2FN~C@ zlxZZQtUucDX44_`LWpzMNd(#qv>9kK(B?j9Ge>T=3D4(*e-CkP@~bHI%A1kiH4rlU zg(1%8%bOu;c?914zM6nHpEqxAihH>8d85P}z|DI+a8!Aco5o0pc~oHAq5}~kS_>^< z*J1c}IJ#|TXN1#Wxmw48Jty&B>u{|lq$53hq2|l^POSTESssXU9Z|079n>ORHHgWn z`N3FVBsmtUf1A)Mw$(3B=1EoV%F$F-9i$xuXiWv}0<+Is+wBIWdKhb>F%<}Fc(`o4 z(>)Kj@tqV0vLdWww%-AYqiuYQnce z|H5p3Cv||_LktU3?(Gk>b#QFt5l<4+T7!FB7VO9?hkM7zpq(D6&xLP zg696);?;;w%F3aNI z&Ej^0f7Wu@MojiIX|=0;-&&`5d#&@zgw}i*Z9YtBNm0r-TDhfmrtHUZf?uOxk!l88 zLrC0|QBh-h4beW5N@k89RFX+<+ed0M)+&*>e>%BYD{xfN8CTRE0BF)t8+B$vDMJ*` zG7tOGxKW5X)&0)EoU*VcFgh<-dkI&7YezXW!Yo!!5g^+oBb#!DYeI7?BPM}FU65KiGJ}jpxcqq&GeK2Xd4vToK&zEFXP}P%)VX~FZ7>z7A(GA O3jIHKCWsFXWB~vKG2^QM diff --git a/src/program/lwaftr/doc/images/decaps-queue-to-internet.png b/src/program/lwaftr/doc/images/decaps-queue-to-internet.png index 2cd301e922ac47798d139e0c948026a986630834..74a7e5992cd2262deb74f48e4f030d6a994e4950 100644 GIT binary patch delta 15431 zcmb8WcQ}^+|37?5$);2yBcuo!A=#_!y|R+Mv-dc8yFx}bMMl|s3mF-al)Yt>ovqC7 z=jHSMeDCjZ-~ar6=h1Oo;kwT2e4Vf7c#P+H*nquPhh0+g9{KT?_LGR=g-AxPD=Lzb z+VoGTDmC7H#v4*m>3RPTS*VVK;b5Ligic}QERD;6hQol)ZyF7~;4&3uQe}1Cd0)ia zx89C2n$#$o;OE|PK>MAJiN(p^2R|0w7k@earmfuQe)n(`=T{3Bk^cIT)6H>IUN%FWue9(Z21@&+j$Fl7+o*TSiAmzkeqh#mR~I^XE@a4pULOvKxzLK~7Q8 z(+Z1Tk-x_QHrNBLF?W<$Sy?fk@96M2+-1J}ivP`GGb7_*4a{47JpbJhuC*( z%do$%dCw!wm3uW6-|yDDcklFqP^eA1{sMH%*qDxzQp73~N|Z)SObqk;ZHv*e%r^a& znQ6>{d~&&hx*0)n-^`x+9O_HU_;}K^Fck`gN;@ab2QQ+_n2YZ0lv7Yx;KxTrwih>^ zes#fxkF-b7l3%-KrP&;nl#~=3+j@4MiH~khnDfm2n)vA9;gJ|;59dJLiNw74^;r&e z8`;{}{P7kZRzdj*dhV}vAeeKJJb3UxPEPK-EZ^EX=BRmf4mHo|vZ7ujfkJt>dw5)W ztBM6@xGLI=d6CW5#*=#bH080-r7P1CWzQ%0&)#?I&K>eAS1ish=xC{zp(x$r`n7v{ z?sIKKMd_$NsXj;h4%Ff;n`@)x#_g{hs4sl_`{N_x#x5daP=1*>Ct6(8j(byK`B5vd*K$cW0kOS3h zkH}t-#M>yrr91k)kx-nnxw-k}i}WMO419lo|Nd+xt%b`+`|CHTsr5e=8w6^0+_TN}1Ez*A~tY3rAz0&JEt|>4u&~~h%#uM4Qs}*IzN={B5O=n67q95fTnuQRlljU! zg6HVym?8h^U^=eMl9LcqrC=X@d`^Z6b?qkWv~8Zz%jKRdHw9c2a9(<5Wo2b)NkK*R zHY!T2b)DiCnLOaq;j z*260wmrcygLkq2DTVJ}my26#oD=HEb5ZveGrFrOh>xAj%O;IH!J6Pz|t5@x&8p6ZF zE6d9Gz5i^cOGVt4fBk83r@y~{*@T~;fBb_TIqrF-6hQ}PXGE{m=*5VsOkUIYxYuZz z>6%FZ4v~@Z(qO)pynI-OD=scB2QP05udR-i)e5X$UO{2`{Zze|xs8qY^d~HSZ9_x4 z)NT>iwI}PNjF0Xy4@5>rZq2p#TcSJ5&AxMSa=wg+P?C|Uaa{PC$YX61_O8p*;;osP z8T_%aO%9+l^jGut0*HQBQHlCnhJ?#`Bh;EQGB5r8 zu_hvLV;FI?SMYFgm-;dZj>@#l&4fI6pBa}|RZW%}2gME=UeAIkn3D!oEmna7QzhezPA9pED=v3)^)c$-ecttJH z_lQjV{rmT)$2;AN-Kk{aympiFB>qDw`Zn`>L_7~4))W-5LTb#PjB#36TZifsEbozu zk5$=r#;>ofd3z(?BO^ApwmUmJnFYjjw^&)%HfLI(%A+Fz0_<%}rB04NhK=*|yr*5( z+TMPLi_7ubYfkH-f~bnbZ9F_Y6`L1Clr0<@8XCG)Ho3!kke3+-t1qV1W13i8`(+gs z@3FBF;p5MqSpWJ_vlihn@~QRJg?+V^nUc7@L@w`1)10{2paxIp&z-$KMlgJDRWo>9kEi~ktsfE(Krh`;8j;5w7;k8(Q6!aj#^ zAOCK((_&#^A%9PPcf{OTKB}s!dj9{`oR>!u@amPKB<4V~ zhQEFDW_f=8mg%{!i^U@fq0b{BkPz|?8p(5H0@u(jX4kXKyHgCjw?2p8ctAx(RZ&^F zx3^~U4%ME4&0WPStqg=ZgFyR$2L$*)Vg#I4{OO1 zb#--l3;#VmIdaZ!>q4KqZK_4wiiBq5=9PkH@l1O z88p~Hb22g#$GtT93%-?c@R_ZxzOwQs#e?~d7}xc&-m;Vwnk3{p1qGg#G@d~h)U&Iu zz39lCoE#{Xi}Ul&>tlnXquRGmDxBzi{C~6;f4YL+-XRunf0xjWvQ+hxIX^jLrKY#Qlm#^W?3W5 z2j0H&^54F_xerYf>TM|mQfzE28h!rI^6}#~z&TJxlV88C_CDC&XgDf*%5xqE=XAT1 z9bk450A^??lOQCsQ3CvnMp^1 zcKJv|b91wivGD_&k(&#+RU=h_Sl|R5f8>qlCJ$%NAEM4IP9q|7QOWRE2ePkca#-Qr5Phtcd7`o^ONX%fmi_`@6ft(^KKf7 zjaPLRz3-QnEC3a#sr6(2a58b52RS`XOiYBp(JFrGfp~aOi}=3vUB8KzeEhh<_oUW# zyb5Y;`s46u=PUZEM7_Peu*<)G{ldY)0cHUp+SL5uXOi9X$-3oVzn<#o94_^yj~Ot~ zGcbgPge<*JfBzm$Zfj*V)`u=Y$l7qT4JIWii!o@=hXXc6K(NRl6@kCguXcH70M`leq|yy!FEIf!4Y9 zh)4T&E@n4k`3axW8}=s(xE0044c5A>X34)XGBTQ7U#GrtgQZgzuYYiG;)C5(>r1kP zd-?!?_`UYmZajG26Ct@zh_9_zKTv~M8`UqS-Htwz9t}8zq6i_9xW^hN;Py+<=P>rP z`Fh6miuPRq7F4Y8lc0m zAVoLQ<*<&UB43y#t71`4k;sF9^s125QjK zpybLgzk*r{^~Qnv1%F6zFb5ZxPMy002msJ5zGqQGRzq4q(o6v41E#fP$ca*9HE4Ld zN;(h~71ayzdr!Z17wR5zxSVY>rlPM;O-nlkH3vGBh9;sp%X#6`J79l{i;LDnCD4VXH{34izQ=1cBY(-P;Qf3lkI)x^S+kC5Qxa?fg0a%*+M2RM?mxXfckz zemn#TK?;DOlY>J?XD0(15Nd68HI&Zzg@w4dxO$%>OPmDbG6%m)Vtfb|&BualA>3SO^X$Kqa##g~4UpMQg#y#3Usx~wevwo!8{GB z&lgZGNBiS;uwA1G3&_GkMOtAY^_460)6<_+^iY4(wef=L>wRG5HLt{dN-sV{&urr4 zyzJ>wE|O3-X#V$SY<70`8#iuzuMNarX?@z!)DV#W07ZFzhUFoXUOkr8qw-hwqREoP zj?GyiFZ zNSQ@0o^^)*yvrT$NbH~oqCP+By&kZ$D?ugO)>pk3awVH=q?dIy!t3JW>3^3>?ycw%Uw5Gb6^#3PHwZj)| zOnhv2xso@xxCj~orHnnOPc=#$qvPY{r)8OBklJ7rpTCVi}P# zuYrMqfPer^3x}T6xxAaCxUtSc@OzZ$IB-m^qQ#qEctu}uui^&XR_k*hHo1Yio0c(0 z1;31ow6`FAu?YV~sXv}MI~OMeLs@w_RL4%2D|)KWFju0!hJ4&;nBzbHikV|kvS@CpPlDF|BLw+=ACze{p_y{Vj7m8XZKs|Q5;mz zgsP4#RE!~ojfBqim8dI8{6Hx_@hu| zIh~i3#id0>MeXf3K}(IY_0#DsNd0{6+O_!DSZ>Qc^s4AUvgl1n@#Wt1cEIH|uP#g@ zf7d}X?nx8Vlj=o59^B#O-3J(U`}S?U_H&}Rbb#GKt@*ExY)=%FML} z!lxEb{$E%U6&L5S*hP)eeebfcfP_0O0>Rznf=h&r!=94_31acXhYxsuim=aDmn&2( z?CcRpNC=2TrDbIR^T`^5?{IK%K%J(iZ|>{k0z?j?1uySVSJ$iV#ShlQzkrdfnP5sJ z0YUi(3}boCYnw>bU41d9IA5#Sk@{RahxAQ5k#)#es1oCF z0luGE)AWnL@ZMWc0!{%9SHJp44pJRT3LuzF_Rgg@+&wo4STZsog9-k8(Ju zYv4!xhPz1zS3;k-&vMn?)%EAj0Z`X$c8Wlc&1wG;=A;L9U5h*H1`XfN=rok2{)Fa) z4gCi1(aA}gsYp~TegHWo|1GM-&!0X$1Kq^w=Qn01rf1j9K8IX|t1-%+M|KW4IXUl} zcgagjODihAdifH7GSs|xsb(f7^2z)>Y;5bF{ILOVrgB?GuMA;{G8k@(OJr+jt8?AI z4=4;H<+$9`1D2}d>|7BV8k(DH2}Jm4Z}pDl*MptK;ei3kM~^^G``X#L3fMp)i8qt> z(f?2_xjXyoZ zW`c);`XbSnT!)n=hUwWoJtY|#Xz1A}6!$CGY5+e_Zb2>r2mtiEnivZ;ztzd!2>WcQ z*=%TKMR^6ef-#Pi?SS+a6&0N|ad4Y}w+$Ac14x^N(Fgxe5f=! zNHCLBH+&QkLH%c@U)=B?QOK?q4sYSqRZ$=3zVC?ObpumS?x#pd4iFbb%B5&6foWM04zVx zOStDN(j4$E(n(551x;Y+(|DbS6G-8789)-?-vU1sDpCWCgXR|mmfAt%%6y!L&F9(J z6uwoUD1qX$LgtAF=jZ3YcyVEP20{5100aEnaVNE>?DpwYWkrP~;0xtCuisr?z6ATc zL}RC6)c|YGPW6@T{L^^!6e_caUFL$W`50AJbT|mfZroU%o=(fm z{5-b>Tqr9nQ~F&_rCM87=BGB5g)DbKYMYz(S6PFdwkubHN_G7ZlFKa|P;J*9cz;*m z1E&Yd{So`h!caeknnATRwqJX@X*PO;C2Nywl_Xn43@#jxpX!FoZ zfLi9^6^rFe@Nre${Q$<^^3oF1v7&=RDb*f#aGWVmfkkd$wJ}!a?3R0JY3Z7rLS0}$ zq6l(|K*KE>T+$5Q0Kxflr)bt{6pFe>Mp8`O)^`1OZU%tDsVNaQw(#yn z04bmVig^FwxPQMK(kUb)Mgqv-Ei{N@$$}oLYv=JdVsSFM+M<$o6;wvU-$q7GFE1~z zuNRJ3J)Gj5zcGzJ%2wA%;?;gGqzSRe12zTfNkfBEz3++VmqZza=Kd_9Fd42+|f2tai||rF)3J3wHJD{n-B+4Uy}QWg9C% zkR;`@87To3pXd1x5YIgqzQz?BG!m1L*bU^UKp<4vP2I~aL=OXk@Zja;Eqjo#tB_v> zEevW-Qeq-F4+0M#UbuL%(S1w&Yu?~#?!1sW-dy5$`3Dt}Jcua!YeFh(MqReE;?h&=`EP zgh<=~Lz$h5P5#&FrqFMnrD>e^!o}9ywbou;;Ew!xo9dkZv+engz2wke2p-+5SFd0T zc6E58&=LS7<>c}$`?EmFg3aJbZg>=W4Qdi{8;y~ZPkv7|_gh<9*3{NQm0ew0A{B2q z-j{-1vQcuU$>X{`kg0$81HZolD_ zvf8H0VxX2`Y!lGNlv0H&KzA_rB)6)?rpIvP{=fXdze%5lLc2x)7m|wq^Ups^ubA)N z4IzD>ojsf`MZ*6E+K7(M7$5>51D4~!)Zm~M@~m*p0V_4F(X?l1C@V%oOKbSol|aLc zU4jXo8x$0fb=R+7@9EI_@VqcTUv4A6Fg?BH^kVdY9+YCZ7mA9C&aT>ta)()r^bZ~+ z_>ylRzzpo|OR}@?M6`SDE=9#^u-qUURt>kFoJ9qjZhPKZ-T_lxcv%tkg((IQrk(M< z%c+7M^t`B2FOSmlezisi2M59y2ylN=&<9m@bP^L1q)r#+=Ub5!y2%p5fQ-s`8WUe% z;gh8JcmQm^hrO9{2{3x$E=V(>5too~pHS_|lh?o_HLrtGF*_@cy9^ne?3Zi`fQ_(4 zT}$f&NK~X;;6whLveQy+*iwV9+@(#DKDNn@O-K;&J$@qF*rQ)wTIv8>@%;I{6Gb~) z+mmDDU}t{bgg-r2i1{dWZ@E94uth$RTPFTzo>Tv2hppL1;?j`iV0teJ)YQ~~|JA2e z=sFi{t8o}s0qe}n%IaeaB8^T+cmS12JQ((o!BFkWbCczI=5&yF6B8**X(38dA38nW z&ucBRSudgJ@M%@FGc$Xfy*yNS1EFgLsW0jWgS<}_(~M@ELjZY-+}E^xJHqI;kLTQ2 zHw^_GW?MnBm7E;-5&9U94ysQkMQs?3pt~r+ZlcGE#hP_g>ma7bw6 zaXIhbUj;BvHxHPsxgY$6j-H-K(rr+|LRqD+8JTArIKd{$;f^BZ<|K3`fd3(3)~LHd zZlnb{IyM$G{yXAdRtEDIp5W8XLmEJFgJ=R0Ov~{Mb;9)E0Y;%9Y=JyI+1EY1`V|QJ zAInsXEv=vc1`7C2H(`4i!I^2FgR~Iz+zloL97R24MtGLPgM^yt-%L@76QH& zIPwhW-L=vFp`j|cvCaXPChJ26f#_mXisO&Dk+q3(Z3>|a%gdJF%&4k@Szv5u2Tt^F zuqCjuv9&b|72)16qV+&<5I+BGZKM<(3An#E0cLfgxERL2Y)mCdy`oy4YHZ9fC;>gD zpKmJF9vP8zq{UNGVt)(^y#4n%VYH{Dd(rRF_a-WfX)!`LGIRPf0@wcB)KFJH-WxV> zUSyDWVyE<>U5hg{V&_bkdS|Moq49_S#39G?46gg2o40-b3{gP?F3a)34mxtCC5TKM z3Jr8}AVorgf=l;0+S?aF-COhD^<+a9z}PZw3p2cK20h)t_rTQ9a0ZP>hD*l-K1rah ziODV~r$8N`)I+@{K(1W9`iP`kb$@@~Wk@so&=p^ZLJarakxX21r7(~30gA5r?C0~qIq3HIp)`1|hOy<4PLqw2-wsUAZ*ySOMt0D9_e zG>GQ3B8acvA0LbW19{AVUi(p_W+GNbD)2EN0Q$?7; zCE((KvIJ=9xV&4K0S*3EVMG4yDh*57E!f8?oV=WD5 zt$MK9I=5x=jn@RJ2+2^8=Cfh;xs{wj6XoDKnjZJQOANUqAS1x|V&lnl0Jl+VFyz#* zFD>51+`K$MXTnLna=n0V5B0r&etQFp`MtA~ZmfD4geuST+h7@hNbch5+V%Bo-_TIX z1G|(>lXL!QkUf|_Ljq84$}?P5ro8_KR_}N??$i~FrhhJYdU{e&P=E&yjX}U`A7PDX zPr7gZib`&4YwMlCHb!@torQUwcEt@Svr6K$eD?K^30N4YrUlvAZlF*C^R__%aQ0OoZk%c4! zQ(^^14d4qw=X?3K_ynpjxWD6Yw&-XiIxLJ3+$_&HCOIvw-+<9T)`KA{V%RaRJ5*9Dyo@-@J?sp@YRC5EgEFG!l2)`gB!k7P+T)8Rl43j5T8q$&RE0U^pr z2&_SfOZ>);tm0xj8=Ez|#*?fOD=fdy9N*X*j~zkZ@%;Y6nh)g%0~=J;5GUYRLDL06 z54AD_q6Y!_y|A#b-TnQd;V@}FlW6e43bgNIrZDZ_3VCU&C>g{Qy?-yRtDA%fn^B`C zr$B0i%B5Q@m8!~@Ngo4JYZ$GFFeBp&;OfGpc^$mCbdUxh-;6f+2sG$nF`+?Eq8h~! zYo<#FR#xz|-V+QXj3?55u4j{jFeWl`bG5iJ&$k!2t;q2vlyK^{Fp8Lzl%hivin115 z9H3oaPSX%c8GG4GUucb)A`hrglE*Mu0$m!9ghXppg-8ZO8;1^DzqP*pe&B;P&!5v= zy$a)n$0?OCo;w9#pGL&@N3ns(4@N@K5TJLJfC5*UP9<)V!u9*E4CGex5{QO?VFc1H z5QG;4f#|tPKJ+;mS=A(3ee9e3kDjd2zkW=JAmo9Y$aMJCntYV4!cRg!`!N> zeYYtesZLODV4g0`R9}Aru1Ke-pg^zMo{kOZ=)UH6Sa#?&w)dvk*a1MtkX}b^?a^pP zIg1*V^v56#ROqRymf4JQ>F{!KC32bKvo-o0xq@6*VPj;(1zxt!y?gh}QX7oT%y?lU zLdO@Ql8Ky3cS50`^4=k01B2^iw_m9OPz41wwL&Khxxj7AG=*_d%v_+OukR@g3RsM` zwV44#t3ZxHh=%CvFH26I0JdiN@WBH&m;tX?o1P{GsM86z6*^PJ+R_rA4VuG5Q%|6DRQ!OR z31y~#8zw(7Lqs_$493QDkZID0ut|WLw)O$sE12(fa@qkqiA)@H?zYM49Jl~z9~Gdu zY(l^2akRI$hqSglfY~_!16d1j)+)>Xz<7{r-oAb<-Fb3|?CorCzh8i5Kp;lM1B2LD zUY6;ES7dV;0^H?{j9WT1+!mx>wN|j~2pXaH803l>rHY4dJ^8a`tOL`i z)_?}sHpVJLPZlPCG>rZgg1av*Czrlp*b?|qK%jrS%61$Yn8ibS`u7li6-XVx6%gfg zXgG}|VcN$6W+A?>B{MNFFpRcu1K8MtR$3uwHNeDcI8kCK0nCjJzC5vh5CU)2pw3;a z6Gnz%B>de1O#3V>F7{&#;L)J*Lj8~NK4rfEO2gp4g$Im4ubD;wTejh?@!VU%(AI4j z4H!Ypw+eJB^Qx;~$3v<^jEi@I%jXWxB6y zGu-?d`_pfeByNDr)&~8d;sVIeSb%)nXIX_YRzNzI2ViaipUGN)MU}dB1I{8&66#JC zNVo@UK)8kM!%#3qRi1yPTpuDZFt|q;Cvj;u^q(^18cAa}Em?$tt1XI6&n~$M4^+mwDvk;}! zUyF<8pehv?Z~XkZ56}q2gz_CgYcVNRseBIafr+Gc|K8q)5q??$F%6AD5|QH`C}EGo zDc>X}!gndX=J5BeaNjbbiAYFHyh|09l%xv?k+HCN*WC=_4oZi(G(cO>G9ubTgM)9* z7O^ogFCNrBHUg7NAq4|aB^STF>4#~1@Ml0@0-7STj?d{s=O`4; zy2?`DXkx>u&7aJe+q+_uIVL%#?Rj5TyJ+{1WhL~@m*L?qj*hcC2OnM6^PC)9hFBSC zP|ug`slh)09|1%&Cr3wsjKi=1AjdB2$e0F*?Evi+m0@YXreVk`Fepe$N(w4wdwV-( zG!caKHIZf*2Y{M_Dc2ZB>~OXC!21;5PfRNe+}=W5+SBt85ZOh->!g=1(UIaRrU*WS zFQ8t!>0+F)O2 z|Hx+*N&)x?QZMYH8))ayoNuB*9vv#msIIPt#LvymWq2I6xwi+SHTy8`;0aj@W*kIC zY0M)jsiT7fc*^2s_aeJ~EjO5oK+OSzZ|}WeHCbkVp+%qDi@v8cjfjXq z6IeX(@C@AcuMOo83*uj3MQH1y+}w9Bz{rEA7G@%VTY*?GA{gnWsE%36gN>`0rL7L=FgO zk&&eSqV_vK6=s$}4)2=VvUt-zM}*J{*OioPLR6tiK`RF^1bs+}E}zv{un%_o%h#_k z;;gbJ03sZi0uP;+A~#KTaftu}uKoNOA017f+r@t*eywab!Rjs@*9X<)mBFyGcJ{iC zj%vO~m4tpFGqL#g>_S2qEZ6)|AD;38B)Go-#S5+qCUg<2Ze4YC$%y^zj0{+4VE3Y& zj;uH4I0nAm+1WYzz@XlfQ82u`+-W@lyase67%xGAf$;dv1Bh=pEl@RlJUoDTMH^p- zSZJ1^y#c%(X$9#$J$>%yPuJx>M!<>tFnfO`rL(p57I;yr$u9e6L5K{hi<>;pG$bH;{HHT6zZ&h^X07O3liTie@4#i*-hhK5aw3=63k+ur=!>#ad81BEwSpNEDt zLAv7PbO0y?vmowVQS8CZ>;eD$1K02h0Pz0WC|IjDlXW$q%s}$r=jOihQE!Gtb>Wp1 zomEgfM;Kzna_g`D{{)6cZo_vb-C#AL=;>mz;4iy;NQq2P#7`N&*QMP;X{C?rS~;8G)&FSb#-)PL#n#mO%8s-^!hRM z^pzE>{Z%jU6@iVxxU=cs*{2^Bmwm)=bDexv*Vwgn0ArrdS%O4wO;)(RX6`0 zqQj2*+{{DusU99n0l4Iud85+kGycrRAwW!)d-qO%!TgD&nfK}OLtPqp^hZVIQaxqV zXJ2OU)pyUS9+uVBdBeCA78`pG_GCdiZjx%ZHi z+@=-LtetTTz7&Jn<*YYI{8nagFWTEdQELR1)!qFM)Fz!uE3ln2p%knAQFn$h?Oe5N zjB*Ef2zUXsNl-DM#*7UN?1G5voCtOpJRmd=C8N<{?(wr{H&eR-+KFju4ne2EL+Idh zAjznJ{SF6+kDv+yceIi+vbG)vya*3_(L{UzsJ6DY2EZ7MCTOJNqoXg-Xn0~}WONkB z>KdS5PA;yM;bL04`F(+MF>OR^TxlYN3h3$GuP+lVqmh|VXh7MQ~#9dm+Y>FNYnqd!&SYp|7(ZKYy?qM z$_E=ps8*;IlpN3efWt&Z!Dv_?^No)v%zpn!0IdstFZ(s+6(R~?N& zaRzjT;f>vk{WlO>@F;xMAr8pNVW=M_=@!@41To}lJO&<@;m0~*n}r(+l7hkp>B7v+ z7L>eRB`}k-6_S2#Y{27f@4!F;UB%MD5-QPC5nK6NVmY25~L+WKnW2k0qGDV6e*Ebx{*#n zDeIK&U@Z3=Q{HRg_+rV-}_!`{c7*kV;_FNe)sMb^5QRcQ^q@Tu`_q5b+2AE zX1z&YNo?y9szn~b?;|c>9pU0abMf#hZ2WHf zx#~ag<(iu7lixmnn^)Cce~&MwFQ(5WyziPo!Y4C=B}dMVhhf`F_Z_(@RAGq_ywJn? zd(qu76n>9l!a}m#c;grjRF4p`U9Ow zynhcCnc$O;y??ExBqdRmlgGrz$1nC|TxnrR2n|)fejPbCke;-+UOknXmNvi;nkph9 z(#sGuPRuBS#>b=jx!BXw(Glo?Z5vZjA)>9VZNpCgNa~HOkym%BsHCK%yK0u)R%C2! zS`4vS2A&9U`AI&CS^ehC3ll6TR8w1&F@9KdbaY@K)`Tt!MM*|af87g>LMgj1^#;0p zcb1Tnl0u}U`WbLh3S@AoTxlHCBUV2@KT=pQ6uj(F_;{F~1~JC?s>a5~j*bGSU$nhi z(%8>%20o>oAy@SFtre_`iAicwlCl>id_BFCd-w(^3KikKyQFJuOjAxyjQN({GynX| zqc0DjzJLFImreO{3~~mQ86Jp@_s`dDY;1h{_U(&?1pVo2DF*XbNw*)prw5yyoJ4Eo zW4=m(hG3%lB`@^o zr}6O@%;>so6pAH8%5%;3!GrD~e)4e3(wkN^Rivnj@abs;man3s;`OBqD4b?)D(36X zVWFh(-shHkSK+0q%6pO315H4XD(;jnl`H{wZg}`+Us>4-&bh?oWX}Em&Q6U|--`QN z{jWmjk`MWq1~MOqU>~=&MxGCyq>*%;AFFnR=3K?uOYL|_2u`!V znU0Q*gM)*CL1Ja4hqJR#Y#Y=@b#?W<@)7g*o(+BnmnBCA^YuS|{K!C0|Mu1}WQT&>D???}yd zi_PD|VQA^-IyzLMX+<4|g|d8dt_pA)CGHh1pK%+FWc>tS3jEcZY1G&LRj@x#XT$=%OU z`1tsW=pvKK&i3}DmQcd1tSl@ntZbENJ)_4OQUS*`uDr*#?{4y`kDQfMR9AnJh;PeO zuP4SI#Qd$uba7>+bH-W9W2NH$cY)GCe17HK0SZnn_sz+P@$vD|Q7JaI*O{59C&-wv zn#l{wpt+b?(u?<=*523qmicT@b%kb-hK6QgVIll>JRS+-@pc;%Y;t4AsSo^{Z}iQ~ z_^JwMgl&e$#w==_S)`?=V(Q^ z1O&A7q;bxkO-)Vx`chh4T>Ljw>W?2ky!)8s{DeeBWdn{S(%+Vq#dq6e$UZc3_f&s+ zjX+dH#C^2F&YQN`)zb1+N{X?bo}QN0*X}fNai_`am(Hf_O-u0e@sT4G6io5~^>;p= z4bgFS-r8B{!u;3oz}4UX&#PCg+=cpvhSs-lJD(i;)QA7xoT8(lVUTnUVGjSbKBk>6 z%IOG@!V`LI~sIM_swDgT}hlcXz~urLKV@eli%iNFy1!lEMRM-P9kH4Cn;tkjql zPs{NN3B?_UciV8Xv(F^^$xRImP{dX@H8tI8@O|pxh)mXx^!4eAxN?hDu+41k?Xg=F z-z*I>+kLH`aF&QRq)YzKZbEkUU48w8r8Uco$>asY3`Uxo(OFq6ldQx+p&8fzzFJImk7soK8Wo2bygFsi8 z@!n!axVbxed!0W%CVE9yg~LW4WOCU%^mEDMbGJ&DuOyRzm{@L;wo zNf2}Ktp=XWC>Z0cSATiI_S@Urll9$g5|pGzVj3G8r*YEL)8`Y3h=>||_gVyv`HXtf zWy{OTwyrf_@r1@8z|HORdt+jGAb0Pi(rHR=SyYF`fF(eGtMe_Rtnak)#SlwN%ah}u zJVP(V<~s8B!hvRaMk1?pxE%Gatj|9*%7t_z4P@3>DruIy_WSRmJSP zpFg_ejr1=;Pv|VtDI~+`8y<$~=2-iq&Uxm{iHYF;n}3$j)6?_8g9p$HS1qnR<6}Xh zuAJrYuRa(Dj?V|TMjWpuQb-pBGd$z3DL7_ZZ z8JQY0fpP)bN@%9l)w$9H-9e3_utfn!SIb7aXbC%)d-}&AkP7YjrLoRlqkhqMDjASJFI}DWI;9w?MU-#pE7dkq+ zzL8cHfoEsr=}Ds%Co7qkS62yDSa;)PHU;cFV6eU_EIevsLZBg>{P{g~Ucjtrb8KX! z_Wk>W1dp|0GK5NhwuH^*>x=xi1#-l`zP`GrF9X_I7(pW=#+^77B(S%rCz%HZ213ga zysV~AF}Tg|@PjT$I8P^A1i+m(S5;MoTSXjpQA#U^Ps^5!Q|m?CATwpy^P4;jgKA1l z`FC_4dV4d!`J;iWW@%-`<}X}_{QUselSm6Ke%Qg&*H>yXpA6^e)2Ds8+Dv`iW0m*Y zY|8cZ^`+gH9_3!w)FdJxXvrWrSwnb4MTeT3o49sY+zszr!V*L z*W3Hwi)$Qi-&Xj+#KFN4L}eM2Ts~g!lVB#I#Kpm3Gw?=}ot+)}sE3OSS%j*_eE$uj z#P{#tXJyUgTuTNR!L|F;_MGK+B^y~v^%b;OnylacRxlokk%7VR$jDRk@p9x|5A@%s zV=)2eS=+&gElCeiel6lZ%e10lX3q_E`A z^P3k&!wVUoo!Qn%i$IIXO8cC20^7 z9UY;2Utg#vaOC9V6q!~#&a{NO4v|t)?sf?_en}PW8XfiRdd3eh)vBeh>cJSDxFaPs zb?Woy74}0KfNAnZVemoS07jSb-L>6Y8G7>M$vmPuGc&Wjy$vua*%ims;dBMzu6;>? zu1bE6hzMF6i~tV+efj#idQpKIu#(H+@Pdw1mYI2bcd2ilCC9kj#{bvIy~y)|&=7Y2 z9R03-FjfUqWF-=t-^$5Ja9KL|3w8qSW$&jonqBSfwlDzXkGF#J^AS5|XJ>Qs2^dVU z^%wq><`oq^0F3Z8U6$_r`5T6YqIW-^a~+@>73Q*bafSKZVj`FKbY4!cC)38LWgF&| z*XHER7uxWlMg+<7B4(1izP-`Td!sliI@-g-LsVQGT5sd<)B{UcoBNKAF;AYr8bnuD z|E_gg-1<(8gy^`t@50r;cVC9el6INBs;=6{K151K2VH4>X6BBq?FDLTG4~}M*JnS3>@m)v954L@;-g|A%#)kYlr(yiU%xWW5jh}u{&6O$N7ty(AV>3ED zEVOFOLjri>0%bc_syjjX(j^h&dzwuqDS8%hk zwsxsCJ0*?Ok$ccC1x+f#Hxy8)om$v7xfuZj$dPBJ-83yR^JEY%;h8gMFqDf~oRc$G zG5BmB(4vZh=+)tC5R|gFkIz3R39|}b?y6WQolg~Z14}@> zGQ(pSq**0$&b*U-_-k-zNIl~Ru!dZ%mwMVVKrkJF@Bs_*PX#y#ZB0x}tbmgZ^(iO> zn>j@1?%j*YUFKz09k7g7<@p5!h)GDi+}%Uow_s=5)z{Y_A0Hp?A*;#Ho^c%70bPe9 zt*x!WjV>-JDLL6fF=MzTKfnLZJpP2P?W?>zE((eS;5(J~zdQE5Qnnh^H8Z29rv3(< z_DUEroE8ZaHTEOw)o(@F(m0w{mX?>@etbzxOceaEKl`&so^v4P&6_u`U%!Ulgu#Dp zbIo@NP)KzDz(7}5SIQn=5}c0e8eLTyK8AVyRgu@_)}O| zxUisL-|s_Zt9fXs^6lGmELIO5tgoz?z{D{!GP-f& z_sO4!&}Cp(LOaOV5W;d}ko6rL7)U^jRV5x#Ub=M2#>NJnb$ECf5fL#sI0$3M(7*t2 z2X*Aa;vybMA*mgoK3R1pa?=yzu^8y;x*YvlfAQkQ&dv^i;)@}C{QNhHOq9K%P(o{o z1aV3h7V<1CfyX02!7j>|m6eUwxLnk9!9rObZM@(7(%s#W3=pg)v#=1Mqo-e6T2j+D zLmiWVP=JRQru9;)k0JC2D~T&Q=p&uNm5|WTK8BzlD!HZUg-+GlI{j$s9ITqPwKb>`on2jS zZf>yB15Z!CzLyK698gX8AN|)S)8+h4jEy_qhG6SZQc%EMI};|3>Xq7O4ogpGgwsfU zY^)uLO9_xh(BeZqs1WW5-WgQfnAeoQDk;uCZ@>@B&dy%)=eWQtA+Zmw(APJ@0b7RA z>CfTbRKQVDfA$%L_^mDHx+}euK0ZE#p;509|Mn*&!P|%_*_xJa2 znwmm|zAMGNpz1XHdV7VClVfO{TtZF?3MiQsS|rc-`t@@YEVcx48B+y@g%J@1xIY@a zHgDP5=Jj7Wqd;){+NI|4sT#zC_iriVtHS(OQ8|0629;X264YHmp}1K2`JhQPrv9DK%M_d0`lITRR+lQ&*?afT>j^%Fp8i}TEdt-+*=I_ zI)f2nxMurvuALs}s+N|Pp&ReC_H}A^ zcX!}cFoAQ&hp^85o^B4lQF80jDs2oDE$qaH4<8ONgr|--Jp4X2#l+(N_r#xp!I~qP zQ#+mVC;)&Fpt#)YDP_KU_Ur=0sE}T08^|)TqJK+E3m|!To8R_qD*za-QX6gUp3za0 zQl@A+@r|{Sa^OSIdh}<>vD&-4fnk1;YH1b$;sC|3iiRoq^T5wgkoX?ml;6+p&(U;- z4SXm$UqHWORS%A4BLy8N>+>JCJ{>uGoJf*~W}|14tX!Z6*)`K0vG0;(9z)qTy$$); z(eXrbQD4oz{q#3NcUo}ju4D$%x)OteS^%tVWNfS)9h50eo=NN{XQBKZqvt*nw2H<{$X}Acv!X-g1xCbR-Z6+zPJC9^_bV1OG5hS@7uaP)nKn?&5s{I= zIgqPflreJ)6Uc8kgV= z(xu$Qf4&Y1(P^?CKIR5Bxhl?Df6kB0>>=S^k6DWgP)_D%(C+qSnC=@lqj z=T6rzkc)Bg5y-=5)Fd9QVa^35^6bm$IkCWDD9@kI&B=k*#t|QXDt=$Jv}A=^=iSI9l@<}Xi{r>%XZ?C(nEAq(`P^?i8$Y^MukP@6bcMc!F)^&jZ4wx;pdapDo zDM?gR6kyvMX;RHLWbvoJzdzo&3-9cQctod<3{tyuvGV<;JSZW;bGI@l z&>TLX6fhyjim;Un{+j@_5rR*#)dg}7$Z|Uu#nN6TCWct-9QZ+f*VIVJ7$eXj+ob?m z!Vu2;;DNe6^xn$}%8mlQxgmyLGCn1x<1HN`DJ|_*YX$|N9}p1W?=O46cAq#bI73r= zXLpw^*TL3yX{hjgK!;_i@2SxdYH_%~A6M?aF<$%n_58$pudQq(QxB$)dzbg?ML>cc6ZQ{w+;#NOR#rIRC{E3Hq2s8e=|c0TD5=c&8RBdj(mQs0)f-=-uFE z!iuNm<-(SuB?gZoBMH;GwY0U{1o4B~+uKQ{twC1<=524!BWZ1KZx0v|#8UamQ?hoBiSgLSc z=meP_NGA{?aEnOj#IE-z{tpCfv;wgUmXC;u2{`)o0LF{EhX+Uv=p(+&vmkMsH~5N9 z==S2nrq=V?7(bX#(0=~kU}A--j!s--qrA*_qCDdaIe!qkG`uKBp$;-z|Dhl=i3FH( zFd+Wde9_7W4gO74)wjOBryGa@3grr{FC?*$8ik@Qtf?6W)8b$7o@;>laHi}3UQ_4K zPu03X?*jZS;kkBdHBq*`4ao*cP*5=T^!IV@oKC1mMqt3)v9dDI*S~`Cg;1zDxZy&V zE*)&mV4sd^m){$R`9HH7gcmyp2QZTEI65+7d@A@rt*Dq-U2QFfhA5ChgM#9OzD=q$ zFPsW=V;)NO9J0oW8jW!QV4^@rMNJ*wBaMr)J?|H(TQk6gQXt&&mI-1Wb`YBgM4{%5 z%*`J-I)1IWNKe8j(+(XS15yw_Xi&or7+Skbj04mM0N$r2eI7Fbad!65(~Eg@w1moR zxRC5DAt^~kMRj^Q#G0C#HMgy;4ULV}orK_20V&K1vA3Pv}iyWQ(7)Am@(EK$uKp?EZI{$*|=t6U{S*7OgE)v=5>g9E# zXI4g57Boz-L42umS$WI}gP3RM<`@_lJlx%H@_mCIVpvMaomlBvFkFEo4oj?zQMiH&E7&4_ut7U6j zTY_MUHxXWy!$=v-W#gPkFl5f2J*)9{t2r1C;J}R=)$BPEd|B{ zx&WXXfKt4?TC~lWA*okh=n zJ-AYeRh&4X)Rk}B_G+b4AIZJ%Ay`e>YyfQq?n`RD8@LyqU!GqA<+Q4*>RPe@gM5ID zQXUY&eSH!Vk`Ns$t9d9hP~Zy-?*q96AU`oNL3HjMoKR3u5YG9CTtFYYQ3#Fj+fxG$ z&T_Ke=c3)R_o35=1C_(I1P)Clw2!xUoG>x12%3eGlJY9)CsV&dZ8 zff8Hfy=HFwd-4!2n2?YVR0n}%X^{EG$IZ${`Aw_#4-dg=0(^ILZ~z+1HCujHSJ$=m zbr%N^ak`Q2ZaOiCt*x!wa??M68@&4k^`YK-nXpP4=Dd0pC+T!gdxIp%SoIz617Ns$#w^={?TfmQ*7 zNg~s=+s%Sh!(wd8?ChxS=JAl=p2eP2(QA{nTTKdRFsZ;3o4G#{LQYJKA!(q#(vgtF z!tFY%Jq4F-RAj;u0xkUS$&pj@G87vKj!H^OySuxjN*4ZmNw0GvC!Ce+O-fJ_$IiAV9CnJ$at;|vsfqP;L%W0&j0u!cDv~j%azB| zspe*8TjqJ&#Ap^PJG&=%i!fqvzk@)L2MR7aYkK;2{oRNsPZZHGQO{ivLTTGLI;JNl zpQoX*yL&fyOpF=xnudAyDoYE=kYo7!O!3Bz^V;7A2iZosaTL{-&({+|o9-JJ5Ec>n zl;E*4sBU4wGES>cz{K+iM1?*ACyXzpkjDMEF->;;5;G!;vCe zeRt-cBpOmYcya0O=kO}~J9k*p8UALsEZ+v*06(ayQPsNiCNv#t6eZe}sc!1YP~@ zxw#KA^cANwaEEat@H17e8e46@Waf;9>0z@ z?(eq)dqpwJ3wfUAOp4=juwz}%loejzH3PmEItchA?g#TiotCWJTrU{SLClQu0a3Xb zm+qcbBn713{5KE;MivQhHvB=$S0shq)H)#l@Yi(*l%=>?Y5APnLvU|#6dk5MFp#v5 zj*ga_wHk%%J*gPaz;g-EE7Ri3NWW>JrT6Nkj0pDR3Y-VXUL0 zU~~cWk(HGQQd$vP()33!7Pe+S;wXxUiekJD77^=C%GoKW8+sqM@51rCj|p3~JvG3( z1fw-)mv9pId1@*^m&9k!W(7?_9Rt#4KxJ> zo=v7vg03ziE*|+OFF$`7Wa+^|^7m!0?oh-Zg}<0=X*r`kP+}n;q64Y}ORXx}r;4lj zv^J5i?AE;Dg zj~~x7^Z+UcT?f)FkZ8~j=>^lC@+N`O613u9zkb1@u8&sIU9!YE$*ZW?1sH*&$Q<(F zXhZr%fvJfJf!z8}uY>D+fIbLIvP|1L0;yLKx-z)EY$DSfoR{Z4%Z6 zb1dLu#-OtRn}>ED(%Te>y}q_41-_B^h31LEn*dgi015UsrV+#snNf=K$0sKzM@Iu7 ziH0agaPXDCCw~$W5}=djzfHXkf{8LZ=gphhFcb*zxGahB@j({9mIqWjqpU7g*NY?I z=>SD?R&Trwv!vCf>c#D05XSMfVg*$^U0f9ZI-ZO7z3N>2%t6OrbScTxKnoU08RI=ke6H9~K*Y3yMWa<4KD&lj4^K+Y0vf$f!@Z9)ysP z=zSj^4kwgh)DFD;=_?}*O-dXR2axCCk1r`uK`bF49fXn2y4J#AhcFGssTQ%Ym>L{3 z*g@x-UF&FXe=g&5S&{UjYRvIUVF?c}Z?1Mm45_`N<1?D7Cr>a!Z-0M(d_n^JLttMQ zge|#(Cz28p?Qb-9^_@0EH)_uJdYsQIyD3L4JPLqbaIoPY$49@KTvj4{ zF!Q*GPs4%0@Pd4R9}vQ;Q(+<40AB=5tD?Rv<=B2NW;hY}aJxAOgtnujqq$26_Y;Wf z$elR=N)242wx$L&gXNVK0rNTo=gXjpW?jL9wp|~_AmyI-1^H`W(^ehcTunz-Z~g_k z8~{230)qO61~``QLMIhu_ro&^`~5hlBxz2}-_BJbfV2x;*2cNYYf(U{dGISIsMatkg%4 zb$N*^zD!^5Ta0pG_w^^zSc?79=C)Z-p(`Wra`bie^Jv-V)hsNK&rvk-_I}IeXZT-wy1SX*9ssP05fP20GRP#ro%ZP`+$-Go&uiu0D3WmEjp{Z)|!q6_=Nn#}L?&@9qS{ z280i}t&fT_FuYt^S^^A(f$&x!Qh*r??cW!?n7TTtS`&42Qh*))xEW})gK5`_HM=@H z?dRK{0+xpGhZeZvZf-|2VN8~H?=F~&suyW6_I@T&JXJ@yzo&R`H{@T1asLObIPh_- zwA2x6iug%YJ+)=BR<5e%E_!!L^$~7vmp0Z-n>XVxQR=?Akpc#?bXD#=_5?}#U3 z(~w|W`D^{OH?++@-8}o*8I&R^oD8VcJ6{dW&1FG0mCGoJhKs^b-uKWPVeQ%_sm*@+bS*)5Zs} z@y{IbX)DF`mt89>f)f`&NCyLHb1*+4(T;g$pQ)Ksum9Q_g#z06yLq8)t*d!+ftb7K z(?je`f6XBqaLFL9)*{dOEcwT|TSI;Q{EFGXY^ef#<4avv=vZ4-XE2eE$74=G0k4jZ$Dm&w^9| z?rFN2TeB!w8MO33{A6YQ;p6ngVH-C=PJ#1Z0G9tMJKNvSZ$6cboLB>z#AwxnYjDxHr$u{m5Ws~ES)t%9jF8A5_rjYXg15N1SSXJ|RY2hA03ZyKFVCe*E|J5^#2Cx@rxtVv7I56#z_Px` z$~x$2a3%im(6`142eq>E^QYJPsI{FP4LDo&_MLEP)sAJ#n z1X2(^FC-~wX!29UTRA}{W`K+s^d-mz2WiXVJ}QOr4nUcYkB^m=)j(ICjlNTn6z5QO|Y_YOr+r4)O z2@3uVhCIwD`=2F1;QHw3Pz5^BHB3#9;KLxnJl$mD;P|yNbRV*L1=8zi{IDJPVu-Fl zGXo15`YZ?_I|s*L1J7!Mc>~}fybI7+SF&K6H9&DlrkefO_J;UzleJiy=43%_R@NOD zi#4E{N_Zg~(6|ZD z>*`Yd4r4+?TOr}#d;n?y6zfF+K$`(a>(w=8hK73s+VV4Hkb!k@;1jrEU;vs4ZgQ>J z20#nv+WLA3`WDqV+`lgk>;W+gfFukEInE=KlL7s&)oaX5OzJ?}FoW=F)RQO83lLa` z)iv{Ubxjqzb4EZOIq`#DSOZqM3{Z440Un;whKJ$;)pi3MJUj%+gaib?HYOsw1Ox?u z*#FPo_nH?;Ngfa(H~Y1565NmvfoL5jfi4_uCt~ zQFFZ1mz}_=JzG{%;$&;fB_IznNUZ!xleN5jBZfHdK#xy&@gl10xTs2geYm7MS<2nUYWZ)QUdNZA0tQyLr>r z&TgdehG0l9?9mQ7dLkIuFs+jTeq)0FX2kgT%l$co0#G4`pi{EaIx&-jZw0>RA}J$N4{*Q64~XqfvKP2-^MBJ}?H3moW_I8UK{asp0ThOdi(3Qv z<{{u1W|K(6LoZ!jG67i151<~@)I(f&M=PF}&aoB|bXPtcj^v{?onQ>%Zk$2nFJV*`U&JOp79y)tc?iQVfS&ki>AKuc=o0h8+8^GWh50I zpIANGY&?IM0qnS`uC6XFjdD*i|^P>cNd_e zgNUu1Yg}ocR9%f!11y1723aH+R*kc<1qB5^y3-anH=V4lTW6fdCnk{nnXor+I9w+| zB%2=iC2aG>+1(wi8Ymnn0d`JX2Zv)oT#zug0(aceabso%Bds6zL7d?xpOK?u@oga( zAW+c>6(9QN)URKshi0N=+aAZlM~y_bS^}tKarsUSnak^GV%?wvH#JQsXv?Ksw(ea0 zUJ`1w0x?Gb1dao5;8B{uw6rl``8l4U&p(jXr9bXG)Yue^;8)2RP>-k8wBdb1ZhvpN zo6EYkf39r^fTNV0+~mkeO1BM@jJM8@S*ve+?2M?T991vaWPm*pm@I{_Ut6sV=D(Ek z&`?)j9;+^aG&1mm^i0^5$lnuj$8k*FFgm)ZxL8X=V`Zd#5o)9Z36Wvk1nr{6Wljaw zy7Ax#omko*u%{qEu9+%=&ytvt(Fn>c*+rGB%F5dZ$AsqvV3PeESn6fjc?lyHlO|>gsKw;?dqJLV$A{COo1X5icCy%c385{P z5CD$25P)?29145<^JkUjSO91qh`W2bJQ8EEG1bV#0>%MkthS~7D3I~{B?7()-m|4t@5o1Kz;(-aw?KI4z5u2ad+6dDX>L~f->w_A%sB7vp8WM@jBP6{DIp;uTKIVYnRbNYMZkEC z3m-LMvN7K`*h?U)IE~i?l3o@UM}Wu8wZ$ftbyChf-2kv?Cg25wYF-+$3Kf-=vVirb zt@hDGHum-_8yo2f2^aIVQv4NhlTP%s*~49VJJ@ar>&03)CuG-1SXZ7y*`LK+1K>!| zfO%`iAe-vtj5sq}$iWvC7k};T)p4IJDB$s{kca#sNE;4fru}*VKi~zf+Un|Rhxim& zBofi^%-vGg@v&b>FHC8GVerTrpliq{OhIZf^zma*Mmpb@xwc+3S5yoHcQmDjIRsRA z=q37kdgvhLu>o+OoCSo1%j=PP8Fi1&HA0@e-DF=L#Y9tej@nc*-IzwO;CBSHjQO!2 zr9xw`0Fnj$W}YfD2%Z~iw|jm3^Jb;}5DV69LtV2!nkkn#?TsiGv}|*Q+Ir(Ca#btsPVFU)SQwZ@ zY2+3r)OV|03Q*@ROKi0>=>i9ojs8&ppem@gAet)xH0|v2i;DOIMgXBemH-5c3l}b^ z6@2RM=9iX+1i0cFqaWO0@U&E8=vk(GK#*Uu0&^!wa68tz{UZP_YEX5+f3i8Jgh@k# zqKmPOpn+^x1{fL|&O?GY8W_1$oi}52?KJ<@-k8e6cdkoQd!2p*Db znD#>cK(#_ZQP8YL$M!Q&==cS23=9knX9~&G=aZGFBSQt5TLt0fLmT9xRoBxa_tDF) ztMi3T56A+Z*8oNa5I9I5jK55z=TS{bEqIG7leC(O2fxH7-80tY#0@)MqKQ@(n2{Dc2{K)2l7+?cjFoxcQ`bxfSVxEvlSlZH73O1K~&U%mTs zJUj&iSYulR#+C%=7N?5W#zbZ&Gh{KIKGj7G3?L;T8Pfai&PStSGa8IlM++IkT6ma) zf`Xz?4RAJWa%u{SkCvsL8{@`iX7%u_7;%`sg@u!o6B8}%%IfNUC#SzaNI<{@5#DO} z-5qom^q=vuv4`tB;D)#tg@RcHZ?z-=BsM$SS@g4&s-RLz?R z01Xqy0<`X?@88Aw`5!=F4Di>-#x9+xFCQ|?K3qPE)QHqwc4q-_Kvi_rx|ZOR_R}w^ z`3^FuUWN6>NGydY*orW5z;}S;JqS>cuJ-lzMxRiYgRcOF8OY$U^Wg>O1j%~@`LmaN zn;mvDfHaVfE~a&V7C#yN;4giW2Y`%)r5Uz6=tZrq)_|n Date: Tue, 8 Nov 2016 15:27:26 +0000 Subject: [PATCH 151/631] Fix YANG cltable serialization --- src/lib/yang/data.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 34a1c0d25f..12df6916f6 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -425,6 +425,16 @@ local function data_printer_from_grammar(production) file:write(indent..'}\n') end end + elseif production.key_ctype then + return function(data, file, indent) + for key, value in cltable.pairs(data) do + print_keyword(keyword, file, indent) + file:write('{\n') + print_key(key, file, indent..' ') + print_value(value, file, indent..' ') + file:write(indent..'}\n') + end + end else return function(data, file, indent) for key, value in pairs(data) do From cba8cee1aa9dcc7eee0e0dee93efcde2116f06f0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 8 Nov 2016 17:19:52 +0100 Subject: [PATCH 152/631] Rename conf.load_lwaftr_config to load_legacy_lwaftr_config --- src/apps/lwaftr/conf.lua | 11 ++++++++--- src/apps/lwaftr/dump.lua | 2 +- src/apps/lwaftr/lwaftr.lua | 2 +- src/program/lwaftr/bench/bench.lua | 2 +- src/program/lwaftr/check/check.lua | 2 +- .../migrate_configuration/migrate_configuration.lua | 4 ++-- src/program/lwaftr/soaktest/soaktest.lua | 2 +- src/program/snabbvmx/lwaftr/lwaftr.lua | 2 +- src/program/snabbvmx/lwaftr/setup.lua | 4 ++-- 9 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/apps/lwaftr/conf.lua b/src/apps/lwaftr/conf.lua index 8abc74620b..a0469c93b4 100644 --- a/src/apps/lwaftr/conf.lua +++ b/src/apps/lwaftr/conf.lua @@ -100,10 +100,14 @@ local lwaftr_conf_spec = { validate=function(parser, config) end } -function load_lwaftr_config(stream) +function load_legacy_lwaftr_config(stream) return Parser.new(stream):parse_property_list(lwaftr_conf_spec) end +function load_lwaftr_config(stream) + error('not yet implemented') +end + function selftest() print('selftest: conf') local lib = require('core.lib') @@ -123,7 +127,8 @@ function selftest() } end local function test(str, expected) - if not lib.equal(expected, load_lwaftr_config(string_file(str))) then + if not lib.equal(expected, + load_legacy_lwaftr_config(string_file(str))) then error('lwaftr conf parse produced unexpected result; string:\n'..str) end end @@ -197,7 +202,7 @@ function selftest() ipv4_ingress_filter = <%s ]] conf_text = conf_text:format(filter_path) - local conf_table = load_lwaftr_config(string_file(conf_text)) + local conf_table = load_legacy_lwaftr_config(string_file(conf_text)) assert(os.remove(filter_path)) if conf_table['ipv4_ingress_filter'] ~= filter_text then error('lwaftr: filter conf contents do not match; pathname:\n'..filter_path) diff --git a/src/apps/lwaftr/dump.lua b/src/apps/lwaftr/dump.lua index 7d6408ebea..1631225f59 100644 --- a/src/apps/lwaftr/dump.lua +++ b/src/apps/lwaftr/dump.lua @@ -152,7 +152,7 @@ function selftest () return table.concat(lines, "\n") end local function test(conf, expected) - local conf_table = lwconf.load_lwaftr_config(string_file(conf)) + local conf_table = lwconf.load_legacy_lwaftr_config(string_file(conf)) conf = do_dump_configuration(conf_table) expected = remove_leading_spaces(expected) if not equal(conf, expected) then diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 4986149b80..4a84b9a51d 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -248,7 +248,7 @@ LwAftr = {} function LwAftr:new(conf) if type(conf) == 'string' then - conf = lwconf.load_lwaftr_config(conf) + conf = lwconf.load_legacy_lwaftr_config(conf) end if conf.debug then debug = true end local o = setmetatable({}, {__index=LwAftr}) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index fb16d9c800..87e12ea11b 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -29,7 +29,7 @@ end function run(args) local opts, conf_file, inv4_pcap, inv6_pcap = parse_args(args) - local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) + local conf = require('apps.lwaftr.conf').load_legacy_lwaftr_config(conf_file) local c = config.new() setup.load_bench(c, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') diff --git a/src/program/lwaftr/check/check.lua b/src/program/lwaftr/check/check.lua index afe3665ff3..0c2440deea 100644 --- a/src/program/lwaftr/check/check.lua +++ b/src/program/lwaftr/check/check.lua @@ -40,7 +40,7 @@ function run(args) or setup.load_check local conf_file, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap, counters_path = unpack(args) - local conf = lwconf.load_lwaftr_config(conf_file) + local conf = lwconf.load_legacy_lwaftr_config(conf_file) local c = config.new() load_check(c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap) diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index f1f65edef3..2b56d8c3c5 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -7,7 +7,7 @@ local ipv4 = require('lib.protocol.ipv4') local ctable = require('lib.ctable') local binding_table = require("apps.lwaftr.binding_table") local conf = require('apps.lwaftr.conf') -local load_lwaftr_config = conf.load_lwaftr_config +local load_legacy_lwaftr_config = conf.load_legacy_lwaftr_config local ffi_array = require('lib.yang.util').ffi_array local yang = require('lib.yang.yang') @@ -110,7 +110,7 @@ end function run(args) binding_table.verbose = false local conf_file = parse_args(args) - local old_conf = load_lwaftr_config(conf_file) + local old_conf = load_legacy_lwaftr_config(conf_file) local new_conf = migrate_conf(old_conf) yang.print_data_for_schema_by_name('snabb-softwire-v1', new_conf, io.stdout) main.exit(0) diff --git a/src/program/lwaftr/soaktest/soaktest.lua b/src/program/lwaftr/soaktest/soaktest.lua index ad4106288d..c9fecce214 100644 --- a/src/program/lwaftr/soaktest/soaktest.lua +++ b/src/program/lwaftr/soaktest/soaktest.lua @@ -39,7 +39,7 @@ function run (args) local load_soak_test = opts["on-a-stick"] and setup.load_soak_test_on_a_stick or setup.load_soak_test local c = config.new() - local conf = lwconf.load_lwaftr_config(conf_file) + local conf = lwconf.load_legacy_lwaftr_config(conf_file) load_soak_test(c, conf, inv4_pcap, inv6_pcap) engine.configure(c) diff --git a/src/program/snabbvmx/lwaftr/lwaftr.lua b/src/program/snabbvmx/lwaftr/lwaftr.lua index 3cdce60cd9..1e0556f220 100644 --- a/src/program/snabbvmx/lwaftr/lwaftr.lua +++ b/src/program/snabbvmx/lwaftr/lwaftr.lua @@ -106,7 +106,7 @@ function run(args) if not file_exists(conf.lwaftr) then fatal(("lwAFTR conf file '%s' not found"):format(conf.lwaftr)) end - lwconf = require('apps.lwaftr.conf').load_lwaftr_config(conf.lwaftr) + lwconf = require('apps.lwaftr.conf').load_legacy_lwaftr_config(conf.lwaftr) lwconf.ipv6_mtu = lwconf.ipv6_mtu or 1500 lwconf.ipv4_mtu = lwconf.ipv4_mtu or 1460 else diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index 21acf920cc..aab3f54731 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -294,10 +294,10 @@ local function load_conf (conf_filename) if not file_exists(filename) then filename = lib.dirname(conf_filename).."/"..filename end - return require("apps.lwaftr.conf").load_lwaftr_config(filename) + return require("apps.lwaftr.conf").load_legacy_lwaftr_config(filename) end local conf = dofile(conf_filename) - return conf, load_lwaftr_config(conf, conf_filename) + return conf, load_legacy_lwaftr_config(conf, conf_filename) end local function lwaftr_app_check (c, conf, lwconf, sources, sinks) From b10a65b17dfab2baec84550b96a3905ec068b6fd Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 8 Nov 2016 15:40:19 +0100 Subject: [PATCH 153/631] Migrate lwAFTR data plane to use YANG conf This also reworks passthrough mode for Snabb vMX, refactors some binding table internals, and obviously touches the lwaftr data plane. --- src/apps/lwaftr/binding_table.lua | 215 +++--------------- src/apps/lwaftr/conf.lua | 190 +++++++--------- src/apps/lwaftr/dump.lua | 211 +---------------- src/apps/lwaftr/lwaftr.lua | 171 +++++++------- src/lib/yang/util.lua | 2 +- .../migrate_configuration.lua | 88 +------ src/program/lwaftr/setup.lua | 199 ++++++++++------ src/program/snabbvmx/lwaftr/lwaftr.lua | 50 ++-- src/program/snabbvmx/lwaftr/setup.lua | 77 +++++-- 9 files changed, 411 insertions(+), 792 deletions(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index 394966b2d6..802c3e5180 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -70,26 +70,16 @@ local lwdebug = require("apps.lwaftr.lwdebug") local Parser = require("apps.lwaftr.conf_parser").Parser local rangemap = require("apps.lwaftr.rangemap") local ctable = require("lib.ctable") +local cltable = require("lib.cltable") +local util = require("lib.yang.util") local band, bor, bxor, lshift, rshift = bit.band, bit.bor, bit.bxor, bit.lshift, bit.rshift -local BINDING_TABLE_MAGIC = "\0bindingtabl" -local BINDING_TABLE_VERSION = 0x00003000 -local binding_table_header_t = ffi.typeof[[ - struct { - uint8_t magic[12]; - uint32_t version; - uint64_t mtime_sec; - uint32_t mtime_nsec; - } -]] - local psid_map_value_t = ffi.typeof[[ struct { uint16_t psid_length; uint16_t shift; } ]] -local br_addresses_header_t = ffi.typeof('struct { uint32_t count; }') -local br_address_t = ffi.typeof('struct { uint8_t addr[16]; }') +local br_address_t = ffi.typeof('uint8_t[16]') -- Total softwire entry size is 32 bytes (with the 4 byte hash), which -- has nice cache alignment properties. @@ -109,38 +99,6 @@ local softwire_value_t = ffi.typeof[[ local SOFTWIRE_TABLE_LOAD_FACTOR = 0.4 -function maybe(f, ...) - local function catch(success, ...) - if success then return ... end - end - return catch(pcall(f, ...)) -end - -local function read_magic(stream) - local header = stream:read_ptr(binding_table_header_t) - local magic = ffi.string(header.magic, ffi.sizeof(header.magic)) - if magic ~= BINDING_TABLE_MAGIC then - stream:error('bad magic') - end - if header.version ~= BINDING_TABLE_VERSION then - stream:error('bad version') - end -end - -function has_magic(stream) - local res = pcall(read_magic, stream) - stream:seek(0) - return res -end - -function is_fresh(stream, mtime_sec, mtime_nsec) - local header = stream:read_ptr(binding_table_header_t) - local res = header.mtime_sec == mtime_sec and header.mtime_nsec == mtime_nsec - stream:seek(0) - return res -end - - BTLookupQueue = {} -- BTLookupQueue needs a binding table to get softwires, BR addresses @@ -199,12 +157,10 @@ end local BindingTable = {} -function BindingTable.new(psid_map, br_addresses, br_address_count, - softwires) +function BindingTable.new(psid_map, br_addresses, softwires) local ret = { psid_map = assert(psid_map), br_addresses = assert(br_addresses), - br_address_count = assert(br_address_count), softwires = assert(softwires) } return setmetatable(ret, {__index=BindingTable}) @@ -245,8 +201,7 @@ function BindingTable:lookup_psid(ipv4, port) end function BindingTable:get_br_address(i) - assert(i= self.br_address_count then return end - return self.br_addresses[idx].addr + if idx > #self.br_addresses then return end + return self.br_addresses[idx] end return next_br_address end @@ -298,77 +253,6 @@ function BindingTable:iterate_softwires() return self.softwires:iterate() end -function BindingTable:save(filename, mtime_sec, mtime_nsec) - local out = stream.open_temporary_output_byte_stream(filename) - out:write_ptr(binding_table_header_t( - BINDING_TABLE_MAGIC, BINDING_TABLE_VERSION, - mtime_sec or 0, mtime_nsec or 0)) - self.psid_map:save(out) - out:write_ptr(br_addresses_header_t(self.br_address_count)) - out:write_array(self.br_addresses, br_address_t, self.br_address_count) - self.softwires:save(out) - out:close_and_rename(filename) -end - -function BindingTable:dump(filename) - local tmp = os.tmpname() - local out = io.open(tmp, 'w+') - local ipv4, ipv6 = require('lib.protocol.ipv4'), require('lib.protocol.ipv6') - local function fmt(out, template, ...) out:write(template:format(...)) end - local function dump(template, ...) fmt(out, template, ...) end - - local function ipv4_ntop(addr) - return ipv4:ntop(ffi.new('uint32_t[1]', { ffi.C.htonl(addr) })) - end - - dump("psid_map {\n") - for lo, hi, psid_info in self:iterate_psid_map() do - dump(" ") - if lo < hi then dump('%s-', ipv4_ntop(lo)) end - dump('%s { psid_length=%d', ipv4_ntop(hi), psid_info.psid_length) - if psid_info.shift ~= 16 - psid_info.shift then - dump(', shift=%d', psid_info.shift) - end - dump(" }\n") - end - dump("}\n\n") - - dump("br_addresses {\n") - for addr in self:iterate_br_addresses() do - dump(" %s\n", ipv6:ntop(addr)) - end - dump("}\n\n") - - dump("softwires {\n") - for entry in self:iterate_softwires() do - dump(" { ipv4=%s, psid=%d, b4=%s", ipv4_ntop(entry.key.ipv4), - entry.key.psid, ipv6:ntop(entry.value.b4_ipv6)) - if entry.value.br ~= 0 then dump(", aftr=%d", entry.value.br) end - dump(" }\n") - end - dump("}\n\n") - - out:flush() - - local res, err = os.rename(tmp, filename) - if not res then - io.stderr:write("Failed to rename "..tmp.." to "..filename..": ") - io.stderr:write(tostring(err).."\n") - else - print("Binding table dumped to "..filename..".") - end -end - -local function load_compiled(stream) - read_magic(stream) - local psid_map = rangemap.load(stream, psid_map_value_t) - local br_address_count = stream:read_ptr(br_addresses_header_t).count - local br_addresses = stream:read_array(br_address_t, br_address_count) - local softwires = ctable.load( - stream, { key_type = softwire_key_t, value_type = softwire_value_t }) - return BindingTable.new(psid_map, br_addresses, br_address_count, softwires) -end - local function parse_psid_map(parser) local psid_info_spec = { parse={ @@ -421,9 +305,12 @@ local function parse_br_addresses(parser) parser:skip_whitespace() if parser:check(',') then parser:skip_whitespace() end end - local ret = ffi.new(ffi.typeof('$[?]', br_address_t), #addresses) - for i, addr in ipairs(addresses) do ret[i-1].addr = addr end - return ret, #addresses + local ptr = ffi.new(ffi.typeof('$[?]', br_address_t), #addresses) + local ret = util.ffi_array(ffi.new(ffi.typeof('$[?]', br_address_t), + #addresses), + br_address_t, #addresses) + for i, addr in ipairs(addresses) do ret[i] = addr end + return ret end local function parse_softwires(parser, psid_map, br_address_count) @@ -482,11 +369,11 @@ end local function parse_binding_table(parser) local psid_map = parse_psid_map(parser) - local br_addresses, br_address_count = parse_br_addresses(parser) - local softwires = parse_softwires(parser, psid_map, br_address_count) + local br_addresses = parse_br_addresses(parser) + local softwires = parse_softwires(parser, psid_map, #br_addresses) parser:skip_whitespace() parser:consume(nil) - return BindingTable.new(psid_map, br_addresses, br_address_count, softwires) + return BindingTable.new(psid_map, br_addresses, softwires) end function load_source(text_stream) @@ -498,46 +385,25 @@ local function log(msg, ...) if verbose then print(msg:format(...)) end end -function load(file) +function load_legacy(file) local source = stream.open_input_byte_stream(file) - if has_magic(source) then - log('loading compiled binding table from %s', file) - return load_compiled(source) - end + return load_source(source:as_text_stream()) +end - -- If the file doesn't have the magic, assume it's a source file. - -- First, see if we compiled it previously and saved a compiled file - -- in a well-known place. - local compiled_file = file:gsub("%.txt$", "")..'.o' - - local compiled_stream = maybe(stream.open_input_byte_stream, - compiled_file) - if compiled_stream then - if has_magic(compiled_stream) then - log('loading compiled binding table from %s', compiled_file) - if is_fresh(compiled_stream, source.mtime_sec, source.mtime_nsec) then - log('compiled binding table %s is up to date.', compiled_file) - return load_compiled(compiled_stream) - end - log('compiled binding table %s is out of date; recompiling.', - compiled_file) - end - compiled_stream:close() +function load(conf) + local psid_builder = rangemap.RangeMapBuilder.new(psid_map_value_t) + local psid_value = psid_map_value_t() + for k, v in cltable.pairs(conf.psid_map) do + local psid_length, shift = v.psid_length, v.shift + shift = shift or 16 - psid_length - (v.reserved_ports_bit_count or 0) + assert(psid_length + shift <= 16, + 'psid_length '..psid_length..' + shift '..shift.. + ' should not exceed 16') + psid_value.psid_length, psid_value.shift = psid_length, shift + psid_builder:add_range(k.addr, v.end_addr or k.addr, psid_value) end - - -- Load and compile it. - log('loading source binding table from %s', file) - local bt = load_source(source:as_text_stream()) - - -- Save it, if we can. - local success, err = pcall(bt.save, bt, compiled_file, - source.mtime_sec, source.mtime_nsec) - if not success then - log('error saving compiled binding table %s: %s', compiled_file, err) - end - - -- Done. - return bt + local psid_map = psid_builder:build(psid_map_value_t()) + return BindingTable.new(psid_map, conf.br_address, conf.softwire) end function selftest() @@ -584,21 +450,6 @@ function selftest() } ]])) - local tmp = os.tmpname() - map:save(tmp) - map = load(tmp) - os.remove(tmp) - - local tmp = os.tmpname() - map:save(tmp) - map = load(tmp) - os.remove(tmp) - - local tmp = os.tmpname() - map:dump(tmp) - map = load(tmp) - os.remove(tmp) - local ipv4_protocol = require("lib.protocol.ipv4") local ipv6_protocol = require("lib.protocol.ipv6") local function pton_host_uint32(ipv4) diff --git a/src/apps/lwaftr/conf.lua b/src/apps/lwaftr/conf.lua index a0469c93b4..21e694bf90 100644 --- a/src/apps/lwaftr/conf.lua +++ b/src/apps/lwaftr/conf.lua @@ -1,8 +1,12 @@ module(..., package.seeall) local ethernet = require("lib.protocol.ethernet") +local ffi = require("ffi") local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") +local util = require("lib.yang.util") +local yang = require('lib.yang.yang') +local cltable = require('lib.cltable') local Parser = require("apps.lwaftr.conf_parser").Parser policies = { @@ -100,114 +104,96 @@ local lwaftr_conf_spec = { validate=function(parser, config) end } +function migrate_conf(old) + function convert_ipv4(addr) + if addr then return util.ipv4_pton(ipv4:ntop(addr)) end + end + local external = { + ip = convert_ipv4(old.aftr_ipv4_ip), + mac = old.aftr_mac_inet_side, + mtu = old.ipv4_mtu, + ingress_filter = old.ipv4_ingress_filter, + egress_filter = old.ipv4_egress_filter, + allow_incoming_icmp = old.policy_icmpv4_incoming == policies.ALLOW, + generate_icmp_errors = old.policy_icmpv4_outgoing == policies.ALLOW, + vlan_tag = old.v4_vlan_tag, + error_rate_limiting = { + packets = old.icmpv4_rate_limiter_n_packets, + period = old.icmpv4_rate_limiter_n_seconds + }, + reassembly = { + max_fragments_per_packet = old.max_fragments_per_reassembly_packet, + max_packets = old.max_ipv4_reassembly_packets + }, + next_hop = { + ip = convert_ipv4(old.next_hop_ipv4_addr), + mac = old.inet_mac + } + } + + local internal = { + ip = old.aftr_ipv6_ip, + mac = old.aftr_mac_b4_side, + mtu = old.ipv6_mtu, + ingress_filter = old.ipv6_ingress_filter, + egress_filter = old.ipv6_egress_filter, + allow_incoming_icmp = old.policy_icmpv6_incoming == policies.ALLOW, + generate_icmp_errors = old.policy_icmpv6_outgoing == policies.ALLOW, + vlan_tag = old.v6_vlan_tag, + error_rate_limiting = { + packets = old.icmpv6_rate_limiter_n_packets, + period = old.icmpv6_rate_limiter_n_seconds + }, + reassembly = { + max_fragments_per_packet = old.max_fragments_per_reassembly_packet, + max_packets = old.max_ipv6_reassembly_packets + }, + next_hop = { + ip = old.next_hop_ipv6_addr, + mac = old.next_hop6_mac + }, + hairpinning = old.hairpinning + } + + local binding_table = require("apps.lwaftr.binding_table") + local old_bt = binding_table.load_legacy(old.binding_table) + local psid_key_t = ffi.typeof('struct { uint32_t addr; }') + local psid_map = cltable.new({ key_type = psid_key_t }) + for addr, end_addr, params in old_bt.psid_map:iterate() do + local reserved_ports_bit_count = 16 - params.psid_length - params.shift + if end_addr == addr then end_addr = nil end + if reserved_ports_bit_count ~= 16 then + psid_map[psid_key_t(addr)] = { + end_addr = end_addr, + psid_length = params.psid_length, + shift = params.shift, + reserved_ports_bit_count = reserved_ports_bit_count + } + end + end + + return { + external_interface = external, + internal_interface = internal, + binding_table = { + psid_map = psid_map, + br_address = old_bt.br_addresses, + softwire = old_bt.softwires + } + } +end + function load_legacy_lwaftr_config(stream) - return Parser.new(stream):parse_property_list(lwaftr_conf_spec) + local conf = Parser.new(stream):parse_property_list(lwaftr_conf_spec) + return migrate_conf(conf) end -function load_lwaftr_config(stream) - error('not yet implemented') +function load_lwaftr_config(filename) + return yang.load_configuration(filename, + {schema_name='snabb-softwire-v1'}) end function selftest() print('selftest: conf') - local lib = require('core.lib') - local function string_file(str) - local pos = 1 - return { - read = function(self, n) - assert(n==1) - local ret - if pos <= #str then - ret = str:sub(pos,pos) - pos = pos + 1 - end - return ret - end, - close = function(self) str = nil end - } - end - local function test(str, expected) - if not lib.equal(expected, - load_legacy_lwaftr_config(string_file(str))) then - error('lwaftr conf parse produced unexpected result; string:\n'..str) - end - end - test([[ - aftr_ipv4_ip = 1.2.3.4 - aftr_ipv6_ip = 8:9:a:b:c:d:e:f - aftr_mac_b4_side = 22:22:22:22:22:22 - aftr_mac_inet_side = 12:12:12:12:12:12 - next_hop6_mac = 44:44:44:44:44:44 - binding_table = "foo-table.txt" - hairpinning = false - icmpv4_rate_limiter_n_packets=6e3 - icmpv4_rate_limiter_n_seconds=2 - icmpv6_rate_limiter_n_packets=6e3 - icmpv6_rate_limiter_n_seconds=2 - inet_mac = 68:68:68:68:68:68 - ipv4_mtu = 1460 - ipv6_mtu = 1500 - max_fragments_per_reassembly_packet = 20 - max_ipv4_reassembly_packets = 100 - max_ipv6_reassembly_packets = 50 - policy_icmpv4_incoming = ALLOW - policy_icmpv6_incoming = ALLOW - policy_icmpv4_outgoing = ALLOW - policy_icmpv6_outgoing = ALLOW - v4_vlan_tag = 1092 # 0x444 - v6_vlan_tag = 1638 # 0x666 - vlan_tagging = true - ]], - { - aftr_ipv4_ip = ipv4:pton('1.2.3.4'), - aftr_ipv6_ip = ipv6:pton('8:9:a:b:c:d:e:f'), - aftr_mac_b4_side = ethernet:pton("22:22:22:22:22:22"), - aftr_mac_inet_side = ethernet:pton("12:12:12:12:12:12"), - next_hop6_mac = ethernet:pton("44:44:44:44:44:44"), - binding_table = "foo-table.txt", - hairpinning = false, - icmpv4_rate_limiter_n_packets=6e3, - icmpv4_rate_limiter_n_seconds=2, - icmpv6_rate_limiter_n_packets=6e3, - icmpv6_rate_limiter_n_seconds=2, - inet_mac = ethernet:pton("68:68:68:68:68:68"), - ipv4_mtu = 1460, - ipv6_mtu = 1500, - max_fragments_per_reassembly_packet = 20, - max_ipv4_reassembly_packets = 100, - max_ipv6_reassembly_packets = 50, - policy_icmpv4_incoming = policies['ALLOW'], - policy_icmpv6_incoming = policies['ALLOW'], - policy_icmpv4_outgoing = policies['ALLOW'], - policy_icmpv6_outgoing = policies['ALLOW'], - v4_vlan_tag = 0x444, - v6_vlan_tag = 0x666, - vlan_tagging = true - } - ) - local function test_loading_filter_conf_from_file() - -- Setup the filter conf file. - local filter_path = os.tmpname() - local filter_text = 'some pflang filter string' - assert(lib.writefile(filter_path, filter_text)) - -- Setup the main config file. - local conf_text = [[ - aftr_ipv4_ip = 1.2.3.4 - aftr_ipv6_ip = 8:9:a:b:c:d:e:f - aftr_mac_b4_side = 22:22:22:22:22:22 - aftr_mac_inet_side = 12:12:12:12:12:12 - next_hop6_mac = 44:44:44:44:44:44 - binding_table = "foo-table.txt" - inet_mac = 68:68:68:68:68:68 - ipv4_ingress_filter = <%s - ]] - conf_text = conf_text:format(filter_path) - local conf_table = load_legacy_lwaftr_config(string_file(conf_text)) - assert(os.remove(filter_path)) - if conf_table['ipv4_ingress_filter'] ~= filter_text then - error('lwaftr: filter conf contents do not match; pathname:\n'..filter_path) - end - end - test_loading_filter_conf_from_file() print('ok') end diff --git a/src/apps/lwaftr/dump.lua b/src/apps/lwaftr/dump.lua index 1631225f59..5ddcd9eca9 100644 --- a/src/apps/lwaftr/dump.lua +++ b/src/apps/lwaftr/dump.lua @@ -1,222 +1,17 @@ module(..., package.seeall) -local binding_table = require("apps.lwaftr.binding_table") -local ethernet = require("lib.protocol.ethernet") -local ipv4 = require("lib.protocol.ipv4") -local ipv6 = require("lib.protocol.ipv6") -local stream = require("apps.lwaftr.stream") +local yang = require("lib.yang.yang") local CONF_FILE_DUMP = "/tmp/lwaftr-%d.conf" -local BINDING_TABLE_FILE_DUMP = "/tmp/binding-table-%d.txt" -local write_to_file = require("apps.lwaftr.lwutil").write_to_file - -Dumper = {} - -function Dumper.mac (val) - return ethernet:ntop(val) -end - -function Dumper.ipv4 (val) - return ipv4:ntop(val) -end - -function Dumper.ipv6 (val) - return ipv6:ntop(val) -end - -function Dumper.number (val) - assert(tonumber(val), "Not a number") - return tostring(val) -end - -function Dumper.string (val) - assert(tostring(val), "Not a string") - return val -end - -function Dumper.boolean (val) - assert(type(val) == "boolean", "Not a boolean") - return val and "true" or "false" -end - -function Dumper.icmp_policy (val) - if val == 1 then return "DROP" end - if val == 2 then return "ALLOW" end -end - -local lwaftr_conf_spec = { - aftr_ipv4_ip=Dumper.ipv4, - aftr_ipv6_ip=Dumper.ipv6, - aftr_mac_b4_side=Dumper.mac, - aftr_mac_inet_side=Dumper.mac, - next_hop6_mac=Dumper.mac, - binding_table=Dumper.string, - hairpinning=Dumper.boolean, - icmpv4_rate_limiter_n_packets=Dumper.number, - icmpv4_rate_limiter_n_seconds=Dumper.number, - icmpv6_rate_limiter_n_packets=Dumper.number, - icmpv6_rate_limiter_n_seconds=Dumper.number, - inet_mac=Dumper.mac, - ipv4_mtu=Dumper.number, - ipv6_mtu=Dumper.number, - max_fragments_per_reassembly_packet=Dumper.number, - max_ipv4_reassembly_packets=Dumper.number, - max_ipv6_reassembly_packets=Dumper.number, - next_hop_ipv4_addr=Dumper.ipv4, - next_hop_ipv6_addr=Dumper.ipv6, - policy_icmpv4_incoming=Dumper.icmp_policy, - policy_icmpv4_outgoing=Dumper.icmp_policy, - policy_icmpv6_incoming=Dumper.icmp_policy, - policy_icmpv6_outgoing=Dumper.icmp_policy, - v4_vlan_tag=Dumper.number, - v6_vlan_tag=Dumper.number, - vlan_tagging=Dumper.boolean, - ipv4_ingress_filter=Dumper.string, - ipv4_egress_filter=Dumper.string, - ipv6_ingress_filter=Dumper.string, - ipv6_egress_filter=Dumper.string, -} - -local function do_dump_configuration (conf) - local result = {} - for k,v in pairs(conf) do - local fn = lwaftr_conf_spec[k] - if fn then - table.insert(result, ("%s = %s"):format(k, fn(v))) - end - end - table.sort(result) - return table.concat(result, "\n") -end function dump_configuration(lwstate) local dest = (CONF_FILE_DUMP):format(os.time()) print(("Dump lwAFTR configuration: '%s'"):format(dest)) - write_to_file(dest, do_dump_configuration(lwstate.conf)) -end - -local function bt_is_fresh (bt_txt, bt_o) - local source = stream.open_input_byte_stream(bt_txt) - local compiled_stream = binding_table.maybe(stream.open_input_byte_stream, bt_o) - return compiled_stream and - binding_table.has_magic(compiled_stream) and - binding_table.is_fresh(compiled_stream, source.mtime_sec, source.mtime_nsec) -end - -local function copy_file (dest, src) - local fin = assert(io.open(src, "rt"), - ("Couldn't open file: '%s'"):format(src)) - local fout = assert(io.open(dest, "wt"), - ("Couldn't open file: '%s'"):format(dest)) - while true do - local str = fin:read(4096) - if not str then break end - fout:write(str) - end - fin:close() - fout:close() -end - -function dump_binding_table (lwstate) - print("Dumping lwAFTR binding table...") - lwstate.binding_table:dump(BINDING_TABLE_FILE_DUMP:format(os.time())) + yang.print_data_for_schema_by_name('snabb-softwire-v1', lwstate.conf, + io.open(dest, 'w')) end function selftest () print("selftest: dump") - local lwconf = require("apps.lwaftr.conf") - local equal = require('core.lib').equal - local function string_file(str) - local pos = 1 - return { - read = function(self, n) - assert(n==1) - local ret - if pos <= #str then - ret = str:sub(pos,pos) - pos = pos + 1 - end - return ret - end, - close = function(self) str = nil end - } - end - local function remove_leading_spaces(str) - local lines = {} - for line in str:gmatch("([^\n]+)") do - line = line:gsub("^%s+", "") - if #line > 0 then - table.insert(lines, line) - end - end - return table.concat(lines, "\n") - end - local function test(conf, expected) - local conf_table = lwconf.load_legacy_lwaftr_config(string_file(conf)) - conf = do_dump_configuration(conf_table) - expected = remove_leading_spaces(expected) - if not equal(conf, expected) then - error("lwAFTR's configuration and dumped version don't match") - end - end - test([[ - aftr_ipv4_ip=1.2.3.4 - aftr_ipv6_ip=8:9:a:b:c:d:e:f - aftr_mac_b4_side=22:22:22:22:22:22 - aftr_mac_inet_side=12:12:12:12:12:12 - next_hop6_mac=44:44:44:44:44:44 - binding_table="foo-table.txt" - hairpinning=false - icmpv4_rate_limiter_n_packets=6e3 - icmpv4_rate_limiter_n_seconds=2 - icmpv6_rate_limiter_n_packets=6e3 - icmpv6_rate_limiter_n_seconds=2 - inet_mac = 68:68:68:68:68:68 - ipv4_mtu = 1460 - ipv6_mtu = 1500 - max_fragments_per_reassembly_packet = 40 - max_ipv4_reassembly_packets = 5 - max_ipv6_reassembly_packets = 10 - policy_icmpv4_incoming = ALLOW - policy_icmpv6_incoming = ALLOW - policy_icmpv4_outgoing = ALLOW - policy_icmpv6_outgoing = ALLOW - v4_vlan_tag = 1092 # 0x444 - v6_vlan_tag = 1638 # 0x666 - vlan_tagging = true - ipv4_ingress_filter="ip" - ipv4_egress_filter="ip" - ipv6_ingress_filter="ip6" - ipv6_egress_filter="ip6" - ]], [[ - aftr_ipv4_ip = 1.2.3.4 - aftr_ipv6_ip = 8:9:a:b:c:d:e:f - aftr_mac_b4_side = 22:22:22:22:22:22 - aftr_mac_inet_side = 12:12:12:12:12:12 - binding_table = foo-table.txt - hairpinning = false - icmpv4_rate_limiter_n_packets = 6000 - icmpv4_rate_limiter_n_seconds = 2 - icmpv6_rate_limiter_n_packets = 6000 - icmpv6_rate_limiter_n_seconds = 2 - inet_mac = 68:68:68:68:68:68 - ipv4_egress_filter = ip - ipv4_ingress_filter = ip - ipv4_mtu = 1460 - ipv6_egress_filter = ip6 - ipv6_ingress_filter = ip6 - ipv6_mtu = 1500 - max_fragments_per_reassembly_packet = 40 - max_ipv4_reassembly_packets = 5 - max_ipv6_reassembly_packets = 10 - next_hop6_mac = 44:44:44:44:44:44 - policy_icmpv4_incoming = ALLOW - policy_icmpv4_outgoing = ALLOW - policy_icmpv6_incoming = ALLOW - policy_icmpv6_outgoing = ALLOW - v4_vlan_tag = 1092 - v6_vlan_tag = 1638 - vlan_tagging = true - ]]) print("ok") end diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 4a84b9a51d..14ce7f8200 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -137,6 +137,14 @@ local function get_icmp_payload(ptr) return ptr + constants.icmp_base_size end +-- This function converts between IPv4-as-host-uint32 and IPv4 as +-- uint8_t[4]. It's a stopgap measure; really the rest of the code +-- should be converted to use IPv4-as-host-uint32. +local function convert_ipv4(addr) + local str = require('lib.yang.util').ipv4_ntop(addr) + return require('lib.protocol.ipv4'):pton(str) +end + local function drop(pkt) packet.free(pkt) end @@ -156,31 +164,23 @@ local function drop_ipv4(lwstate, pkt, pkt_src_link) return drop(pkt) end -local transmit_icmpv6_reply, transmit_icmpv4_reply - -local function init_transmit_icmpv6_reply(lwstate) - assert(lwstate.icmpv6_rate_limiter_n_seconds > 0, - "Incorrect icmpv6_rate_limiter_n_seconds value, must be > 0") - assert(lwstate.icmpv6_rate_limiter_n_packets >= 0, - "Incorrect icmpv6_rate_limiter_n_packets value, must be >= 0") - local icmpv6_rate_limiter_n_seconds = lwstate.icmpv6_rate_limiter_n_seconds - local icmpv6_rate_limiter_n_packets = lwstate.icmpv6_rate_limiter_n_packets +local function init_transmit_icmpv6_reply (rate_limiting) local num_packets = 0 local last_time - return function (o, pkt) + return function (lwstate, pkt) local now = tonumber(engine.now()) last_time = last_time or now -- Reset if elapsed time reached. - if now - last_time >= icmpv6_rate_limiter_n_seconds then + if now - last_time >= rate_limiting.period then last_time = now num_packets = 0 end -- Send packet if limit not reached. - if num_packets < icmpv6_rate_limiter_n_packets then + if num_packets < rate_limiting.packets then num_packets = num_packets + 1 counter.add(lwstate.counters["out-icmpv6-bytes"], pkt.length) counter.add(lwstate.counters["out-icmpv6-packets"]) - return transmit(o, pkt) + return transmit(lwstate.o6, pkt) else counter.add(lwstate.counters["drop-over-rate-limit-icmpv6-bytes"], pkt.length) counter.add(lwstate.counters["drop-over-rate-limit-icmpv6-packets"]) @@ -193,20 +193,14 @@ local function ipv4_in_binding_table (lwstate, ip) return lwstate.binding_table:is_managed_ipv4_address(ip) end -local function init_transmit_icmpv4_reply (lwstate) - assert(lwstate.icmpv4_rate_limiter_n_seconds > 0, - "Incorrect icmpv4_rate_limiter_n_seconds value, must be > 0") - assert(lwstate.icmpv4_rate_limiter_n_packets >= 0, - "Incorrect icmpv4_rate_limiter_n_packets value, must be >= 0") - local icmpv4_rate_limiter_n_seconds = lwstate.icmpv4_rate_limiter_n_seconds - local icmpv4_rate_limiter_n_packets = lwstate.icmpv4_rate_limiter_n_packets +local function init_transmit_icmpv4_reply (rate_limiting) local num_packets = 0 local last_time - return function (o, pkt, orig_pkt, orig_pkt_link) + return function (lwstate, pkt, orig_pkt, orig_pkt_link) local now = tonumber(engine.now()) last_time = last_time or now -- Reset if elapsed time reached. - if now - last_time >= icmpv4_rate_limiter_n_seconds then + if now - last_time >= rate_limiting.period then last_time = now num_packets = 0 end @@ -217,7 +211,7 @@ local function init_transmit_icmpv4_reply (lwstate) drop(orig_pkt) end -- Send packet if limit not reached. - if num_packets < icmpv4_rate_limiter_n_packets then + if num_packets < rate_limiting.packets then num_packets = num_packets + 1 counter.add(lwstate.counters["out-icmpv4-bytes"], pkt.length) counter.add(lwstate.counters["out-icmpv4-packets"]) @@ -254,36 +248,24 @@ function LwAftr:new(conf) local o = setmetatable({}, {__index=LwAftr}) o.conf = conf - -- FIXME: Access these from the conf instead of splatting them onto - -- the lwaftr app, if there is no performance impact. - o.aftr_ipv4_ip = conf.aftr_ipv4_ip - o.aftr_mac_b4_side = conf.aftr_mac_b4_side - o.aftr_mac_inet_side = conf.aftr_mac_inet_side - o.next_hop6_mac = conf.next_hop6_mac or ethernet:pton("00:00:00:00:00:00") - o.hairpinning = conf.hairpinning - o.icmpv4_rate_limiter_n_packets = conf.icmpv4_rate_limiter_n_packets - o.icmpv4_rate_limiter_n_seconds = conf.icmpv4_rate_limiter_n_seconds - o.icmpv6_rate_limiter_n_packets = conf.icmpv6_rate_limiter_n_packets - o.icmpv6_rate_limiter_n_seconds = conf.icmpv6_rate_limiter_n_seconds - o.inet_mac = conf.inet_mac or ethernet:pton("00:00:00:00:00:00") - o.ipv4_mtu = conf.ipv4_mtu - o.ipv6_mtu = conf.ipv6_mtu - o.max_fragments_per_reassembly_packet = conf.max_fragments_per_reassembly_packet - o.max_ipv6_reassembly_packets = conf.max_ipv6_reassembly_packets - o.policy_icmpv4_incoming = conf.policy_icmpv4_incoming - o.policy_icmpv4_outgoing = conf.policy_icmpv4_outgoing - o.policy_icmpv6_incoming = conf.policy_icmpv6_incoming - o.policy_icmpv6_outgoing = conf.policy_icmpv6_outgoing - - o.binding_table = conf.preloaded_binding_table or bt.load(o.conf.binding_table) + o.binding_table = bt.load(conf.binding_table) o.inet_lookup_queue = bt.BTLookupQueue.new(o.binding_table) o.hairpin_lookup_queue = bt.BTLookupQueue.new(o.binding_table) + if not conf.internal_interface.next_hop.mac then + conf.internal_interface.next_hop.mac = ethernet:pton('00:00:00:00:00:00') + end + if not conf.external_interface.next_hop.mac then + conf.external_interface.next_hop.mac = ethernet:pton('00:00:00:00:00:00') + end + o.control = channel.create('lwaftr/control', messages.lwaftr_message_t) o.counters = assert(conf.counters, "Counters not initialized") - transmit_icmpv6_reply = init_transmit_icmpv6_reply(o) - transmit_icmpv4_reply = init_transmit_icmpv4_reply(o) + o.transmit_icmpv6_reply = init_transmit_icmpv6_reply( + conf.internal_interface.error_rate_limiting) + o.transmit_icmpv4_reply = init_transmit_icmpv4_reply( + conf.external_interface.error_rate_limiting) if debug then lwdebug.pp(conf) end return o end @@ -308,23 +290,6 @@ local function decrement_ttl(pkt) return new_ttl end --- https://www.ietf.org/id/draft-farrer-softwire-br-multiendpoints-01.txt --- Return the IPv6 address of the B4 and the AFTR. -local function binding_lookup_ipv4(lwstate, ipv4_ip, port) - if debug then - print(lwdebug.format_ipv4(ipv4_ip), 'port: ', port, string.format("%x", port)) - lwdebug.pp(lwstate.binding_table) - end - local val = lwstate.binding_table:lookup(ipv4_ip, port) - if val then - return val.b4_ipv6, lwstate.binding_table:get_br_address(val.br) - end - if debug then - print("Nothing found for ipv4:port", lwdebug.format_ipv4(ipv4_ip), - string.format("%i (0x%x)", port, port)) - end -end - -- Hairpinned packets need to be handled quite carefully. We've decided they: -- * should increment hairpin-ipv4-bytes and hairpin-ipv4-packets -- * should increment [in|out]-ipv6-[bytes|packets] @@ -345,7 +310,8 @@ end local function transmit_ipv4(lwstate, pkt) local ipv4_header = get_ethernet_payload(pkt) local dst_ip = get_ipv4_dst_address(ipv4_header) - if lwstate.hairpinning and ipv4_in_binding_table(lwstate, dst_ip) then + if (lwstate.conf.internal_interface.hairpinning and + ipv4_in_binding_table(lwstate, dst_ip)) then -- The destination address is managed by the lwAFTR, so we need to -- hairpin this packet. Enqueue on the IPv4 interface, as if it -- came from the internet. @@ -365,7 +331,7 @@ local function drop_ipv4_packet_to_unreachable_host(lwstate, pkt, pkt_src_link) counter.add(lwstate.counters["drop-no-dest-softwire-ipv4-bytes"], pkt.length) counter.add(lwstate.counters["drop-no-dest-softwire-ipv4-packets"]) - if lwstate.policy_icmpv4_outgoing == lwconf.policies['DROP'] then + if not lwstate.conf.external_interface.generate_icmp_errors then -- ICMP error messages off by policy; silently drop. -- Not counting bytes because we do not even generate the packets. counter.add(lwstate.counters["drop-out-by-policy-icmpv4-packets"]) @@ -387,16 +353,18 @@ local function drop_ipv4_packet_to_unreachable_host(lwstate, pkt, pkt_src_link) code = constants.icmpv4_host_unreachable, } local icmp_dis = icmp.new_icmpv4_packet( - lwstate.aftr_mac_inet_side, lwstate.inet_mac, lwstate.aftr_ipv4_ip, + lwstate.conf.external_interface.mac, + lwstate.conf.external_interface.next_hop.mac, + convert_ipv4(lwstate.conf.external_interface.ip), to_ip, pkt, icmp_config) - return transmit_icmpv4_reply(lwstate, icmp_dis, pkt, pkt_src_link) + return lwstate:transmit_icmpv4_reply(icmp_dis, pkt, pkt_src_link) end -- ICMPv6 type 1 code 5, as per RFC 7596. -- The source (ipv6, ipv4, port) tuple is not in the table. local function drop_ipv6_packet_from_bad_softwire(lwstate, pkt, br_addr) - if lwstate.policy_icmpv6_outgoing == lwconf.policies['DROP'] then + if not lwstate.conf.internal_interface.generate_icmp_errors then -- ICMP error messages off by policy; silently drop. -- Not counting bytes because we do not even generate the packets. counter.add(lwstate.counters["drop-out-by-policy-icmpv6-packets"]) @@ -412,15 +380,17 @@ local function drop_ipv6_packet_from_bad_softwire(lwstate, pkt, br_addr) code = constants.icmpv6_failed_ingress_egress_policy, } local b4fail_icmp = icmp.new_icmpv6_packet( - lwstate.aftr_mac_b4_side, lwstate.next_hop6_mac, icmpv6_src_addr, - orig_src_addr_icmp_dst, pkt, icmp_config) + lwstate.conf.internal_interface.mac, + lwstate.conf.internal_interface.next_hop.mac, + icmpv6_src_addr, orig_src_addr_icmp_dst, pkt, icmp_config) drop(pkt) - transmit_icmpv6_reply(lwstate.o6, b4fail_icmp) + lwstate:transmit_icmpv6_reply(b4fail_icmp) end local function encapsulating_packet_with_df_flag_would_exceed_mtu(lwstate, pkt) local payload_length = get_ethernet_payload_length(pkt) - if payload_length + ipv6_fixed_header_size <= lwstate.ipv6_mtu then + local mtu = lwstate.conf.internal_interface.mtu + if payload_length + ipv6_fixed_header_size <= mtu then -- Packet will not exceed MTU. return false end @@ -436,14 +406,18 @@ local function cannot_fragment_df_packet_error(lwstate, pkt) if debug then lwdebug.print_pkt(pkt) end -- The ICMP packet should be set back to the packet's source. local dst_ip = get_ipv4_src_address_ptr(get_ethernet_payload(pkt)) + local mtu = lwstate.conf.internal_interface.mtu local icmp_config = { type = constants.icmpv4_dst_unreachable, code = constants.icmpv4_datagram_too_big_df, extra_payload_offset = 0, - next_hop_mtu = lwstate.ipv6_mtu - constants.ipv6_fixed_header_size, + next_hop_mtu = mtu - constants.ipv6_fixed_header_size, } - return icmp.new_icmpv4_packet(lwstate.aftr_mac_inet_side, lwstate.inet_mac, - lwstate.aftr_ipv4_ip, dst_ip, pkt, icmp_config) + return icmp.new_icmpv4_packet( + lwstate.conf.external_interface.mac, + lwstate.conf.external_interface.next_hop.mac, + convert_ipv4(lwstate.conf.external_interface.ip), + dst_ip, pkt, icmp_config) end local function encapsulate_and_transmit(lwstate, pkt, ipv6_dst, ipv6_src, pkt_src_link) @@ -452,7 +426,7 @@ local function encapsulate_and_transmit(lwstate, pkt, ipv6_dst, ipv6_src, pkt_sr if ttl == 0 then counter.add(lwstate.counters["drop-ttl-zero-ipv4-bytes"], pkt.length) counter.add(lwstate.counters["drop-ttl-zero-ipv4-packets"]) - if lwstate.policy_icmpv4_outgoing == lwconf.policies['DROP'] then + if not lwstate.conf.external_interface.generate_icmp_errors then -- Not counting bytes because we do not even generate the packets. counter.add(lwstate.counters["drop-out-by-policy-icmpv4-packets"]) return drop_ipv4(lwstate, pkt, pkt_src_link) @@ -463,28 +437,30 @@ local function encapsulate_and_transmit(lwstate, pkt, ipv6_dst, ipv6_src, pkt_sr code = constants.icmpv4_ttl_exceeded_in_transit, } local reply = icmp.new_icmpv4_packet( - lwstate.aftr_mac_inet_side, lwstate.inet_mac, lwstate.aftr_ipv4_ip, + lwstate.conf.external_interface.mac, + lwstate.conf.external_interface.next_hop.mac, + convert_ipv4(lwstate.conf.external_interface.ip), dst_ip, pkt, icmp_config) - return transmit_icmpv4_reply(lwstate, reply, pkt, pkt_src_link) + return lwstate:transmit_icmpv4_reply(reply, pkt, pkt_src_link) end if debug then print("ipv6", ipv6_src, ipv6_dst) end local next_hdr_type = proto_ipv4 - local ether_src = lwstate.aftr_mac_b4_side - local ether_dst = lwstate.next_hop6_mac + local ether_src = lwstate.conf.internal_interface.mac + local ether_dst = lwstate.conf.internal_interface.next_hop.mac if encapsulating_packet_with_df_flag_would_exceed_mtu(lwstate, pkt) then counter.add(lwstate.counters["drop-over-mtu-but-dont-fragment-ipv4-bytes"], pkt.length) counter.add(lwstate.counters["drop-over-mtu-but-dont-fragment-ipv4-packets"]) - if lwstate.policy_icmpv4_outgoing == lwconf.policies['DROP'] then + if not lwstate.conf.external_interface.generate_icmp_errors then -- Not counting bytes because we do not even generate the packets. counter.add(lwstate.counters["drop-out-by-policy-icmpv4-packets"]) return drop_ipv4(lwstate, pkt, pkt_src_link) end local reply = cannot_fragment_df_packet_error(lwstate, pkt) - return transmit_icmpv4_reply(lwstate, reply, pkt, pkt_src_link) + return lwstate:transmit_icmpv4_reply(reply, pkt, pkt_src_link) end local payload_length = get_ethernet_payload_length(pkt) @@ -626,7 +602,7 @@ local function from_inet(lwstate, pkt, pkt_src_link) -- than other protocols. local ipv4_header = get_ethernet_payload(pkt) if get_ipv4_proto(ipv4_header) == proto_icmp then - if lwstate.policy_icmpv4_incoming == lwconf.policies['DROP'] then + if not lwstate.conf.external_interface.allow_incoming_icmp then counter.add(lwstate.counters["drop-in-by-policy-icmpv4-bytes"], pkt.length) counter.add(lwstate.counters["drop-in-by-policy-icmpv4-packets"]) return drop_ipv4(lwstate, pkt, pkt_src_link) @@ -664,9 +640,11 @@ local function tunnel_unreachable(lwstate, pkt, code, next_hop_mtu) next_hop_mtu = next_hop_mtu } local dst_ip = get_ipv4_src_address_ptr(embedded_ipv4_header) - local icmp_reply = icmp.new_icmpv4_packet(lwstate.aftr_mac_inet_side, lwstate.inet_mac, - lwstate.aftr_ipv4_ip, dst_ip, pkt, - icmp_config) + local icmp_reply = icmp.new_icmpv4_packet( + lwstate.conf.external_interface.mac, + lwstate.conf.external_interface.next_hop.mac, + convert_ipv4(lwstate.conf.external_interface.ip), + dst_ip, pkt, icmp_config) return icmp_reply end @@ -690,7 +668,7 @@ local function icmpv6_incoming(lwstate, pkt) local reply = tunnel_unreachable(lwstate, pkt, constants.icmpv4_datagram_too_big_df, mtu) - return transmit_icmpv4_reply(lwstate, reply, pkt) + return lwstate:transmit_icmpv4_reply(reply, pkt) -- Take advantage of having already checked for 'packet too big' (2), and -- unreachable node/hop limit exceeded/paramater problem being 1, 3, 4 respectively elseif icmp_type <= constants.icmpv6_parameter_problem then @@ -709,7 +687,7 @@ local function icmpv6_incoming(lwstate, pkt) -- Accept all unreachable or parameter problem codes local reply = tunnel_unreachable(lwstate, pkt, constants.icmpv4_host_unreachable) - return transmit_icmpv4_reply(lwstate, reply, pkt) + return lwstate:transmit_icmpv4_reply(reply, pkt) else -- No other types of ICMPv6, including echo request/reply, are -- handled. @@ -734,8 +712,11 @@ local function flush_decapsulation(lwstate) -- Source softwire is valid; decapsulate and forward. -- Note that this may invalidate any pointer into pkt.data. Be warned! pkt = packet.shiftleft(pkt, ipv6_fixed_header_size) - write_eth_header(pkt.data, lwstate.aftr_mac_inet_side, lwstate.inet_mac, - n_ethertype_ipv4) + write_eth_header( + pkt.data, + lwstate.conf.external_interface.mac, + lwstate.conf.external_interface.next_hop.mac, + n_ethertype_ipv4) transmit_ipv4(lwstate, pkt) else counter.add(lwstate.counters["drop-no-source-softwire-ipv6-bytes"], pkt.length) @@ -767,7 +748,7 @@ local function from_b4(lwstate, pkt) if proto ~= proto_ipv4 then if proto == proto_icmpv6 then - if lwstate.policy_icmpv6_incoming == lwconf.policies['DROP'] then + if not lwstate.conf.internal_interface.allow_incoming_icmp then counter.add(lwstate.counters["drop-in-by-policy-icmpv6-bytes"], pkt.length) counter.add(lwstate.counters["drop-in-by-policy-icmpv6-packets"]) counter.add(lwstate.counters["drop-all-ipv6-iface-bytes"], pkt.length) @@ -832,7 +813,10 @@ function LwAftr:push () if msg then if msg.kind == messages.lwaftr_message_reload then print('Reloading binding table.') - self.binding_table = bt.load(self.conf.binding_table) + print('FIXME: Out to lunch, back shortly') + if false then + self.binding_table = bt.load(self.conf.binding_table) + end -- We don't know why yet, but something about reloading a -- binding table makes LuaJIT switch to side traces instead -- of main traces. Very weird. Flushing the JIT state @@ -840,7 +824,6 @@ function LwAftr:push () require('jit').flush() elseif msg.kind == messages.lwaftr_message_dump_config then dump.dump_configuration(self) - dump.dump_binding_table(self) else print('Unhandled message: '..tostring(msg)) end diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index afe06c66f9..a181548638 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -50,7 +50,7 @@ function ffi_array(ptr, elt_t, count) assert(1 <= idx and idx <= size) return ptr[idx-1] end - function mt:__setindex(idx, val) + function mt:__newindex(idx, val) assert(1 <= idx and idx <= size) ptr[idx-1] = val end diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 2b56d8c3c5..6bdc5edcbc 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -24,94 +24,10 @@ local function parse_args(args) return unpack(args) end -local function migrate_conf(old) - function convert_ipv4(addr) - if addr then return util.ipv4_pton(ipv4:ntop(addr)) end - end - local external = { - ip = convert_ipv4(old.aftr_ipv4_ip), - mac = old.aftr_mac_inet_side, - mtu = old.ipv4_mtu, - ingress_filter = old.ipv4_ingress_filter, - egress_filter = old.ipv4_egress_filter, - allow_incoming_icmp = old.policy_icmpv4_incoming == conf.policies.ALLOW, - generate_icmp_errors = old.policy_icmpv4_outgoing == conf.policies.ALLOW, - vlan_tag = old.v4_vlan_tag, - error_rate_limiting = { - packets = old.icmpv4_rate_limiter_n_packets, - seconds = old.icmpv4_rate_limiter_n_seconds - }, - reassembly = { - max_fragments_per_packet = old.max_fragments_per_reassembly_packet, - max_packets = old.max_ipv4_reassembly_packets - }, - next_hop = { - ip = convert_ipv4(old.next_hop_ipv4_addr), - mac = old.inet_mac - } - } - - local internal = { - ip = old.aftr_ipv6_ip, - mac = old.aftr_mac_b4_side, - mtu = old.ipv6_mtu, - ingress_filter = old.ipv6_ingress_filter, - egress_filter = old.ipv6_egress_filter, - allow_incoming_icmp = old.policy_icmpv6_incoming == conf.policies.ALLOW, - generate_icmp_errors = old.policy_icmpv6_outgoing == conf.policies.ALLOW, - vlan_tag = old.v6_vlan_tag, - error_rate_limiting = { - packets = old.icmpv6_rate_limiter_n_packets, - seconds = old.icmpv6_rate_limiter_n_seconds - }, - reassembly = { - max_fragments_per_packet = old.max_fragments_per_reassembly_packet, - max_packets = old.max_ipv6_reassembly_packets - }, - next_hop = { - ip = old.next_hop_ipv6_addr, - mac = old.next_hop6_mac - }, - hairpinning = old.hairpinning - } - - local old_bt = binding_table.load(old.binding_table) - local psid_map = {} - for addr, end_addr, params in old_bt.psid_map:iterate() do - local reserved_ports_bit_count = 16 - params.psid_length - params.shift - if end_addr == addr then end_addr = nil end - if reserved_ports_bit_count ~= 16 then - psid_map[{addr=addr}] = { - end_addr = end_addr, - psid_length = params.psid_length, - shift = params.shift, - reserved_ports_bit_count = reserved_ports_bit_count - } - end - end - local br_address_t = ffi.typeof('uint8_t[16]') - local br_address_array = ffi.cast (ffi.typeof('$*', br_address_t), - old_bt.br_addresses) - local br_addresses = ffi_array(br_address_array, br_address_t, - old_bt.br_address_count) - local softwires = old_bt.softwires - - return { - external_interface = external, - internal_interface = internal, - binding_table = { - psid_map = psid_map, - br_address = br_addresses, - softwire = softwires - } - } -end - function run(args) binding_table.verbose = false local conf_file = parse_args(args) - local old_conf = load_legacy_lwaftr_config(conf_file) - local new_conf = migrate_conf(old_conf) - yang.print_data_for_schema_by_name('snabb-softwire-v1', new_conf, io.stdout) + local conf = load_legacy_lwaftr_config(conf_file) + yang.print_data_for_schema_by_name('snabb-softwire-v1', conf, io.stdout) main.exit(0) end diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 4563009642..263113fa08 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -9,54 +9,78 @@ local lwaftr = require("apps.lwaftr.lwaftr") local lwcounter = require("apps.lwaftr.lwcounter") local basic_apps = require("apps.basic.basic_apps") local pcap = require("apps.pcap.pcap") -local bt = require("apps.lwaftr.binding_table") local ipv4_apps = require("apps.lwaftr.ipv4_apps") local ipv6_apps = require("apps.lwaftr.ipv6_apps") local vlan = require("apps.vlan.vlan") +local ipv4 = require("lib.protocol.ipv4") local ethernet = require("lib.protocol.ethernet") +local ipv4_ntop = require("lib.yang.util").ipv4_ntop + +local function convert_ipv4(addr) + if addr ~= nil then return ipv4:pton(ipv4_ntop(addr)) end +end function lwaftr_app(c, conf) assert(type(conf) == 'table') - conf.preloaded_binding_table = bt.load(conf.binding_table) local function append(t, elem) table.insert(t, elem) end local function prepend(t, elem) table.insert(t, 1, elem) end conf.counters = lwcounter.init_counters() - config.app(c, "reassemblerv4", ipv4_apps.Reassembler, conf) - config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, conf) - config.app(c, "icmpechov4", ipv4_apps.ICMPEcho, { address = conf.aftr_ipv4_ip }) - config.app(c, "icmpechov6", ipv6_apps.ICMPEcho, { address = conf.aftr_ipv6_ip }) + config.app(c, "reassemblerv4", ipv4_apps.Reassembler, + { max_ipv4_reassembly_packets = + conf.external_interface.reassembly.max_packets, + max_fragments_per_reassembly_packet = + conf.external_interface.reassembly.max_fragments_per_packet, + counters = conf.counters }) + config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, + { max_ipv6_reassembly_packets = + conf.internal_interface.reassembly.max_packets, + max_fragments_per_reassembly_packet = + conf.internal_interface.reassembly.max_fragments_per_packet, + counters = conf.counters }) + config.app(c, "icmpechov4", ipv4_apps.ICMPEcho, + { address = convert_ipv4(conf.external_interface.ip) }) + config.app(c, "icmpechov6", ipv6_apps.ICMPEcho, + { address = conf.internal_interface.ip }) config.app(c, 'lwaftr', lwaftr.LwAftr, conf) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, - { mtu=conf.ipv4_mtu, counters=conf.counters }) + { mtu=conf.external_interface.mtu, counters=conf.counters }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, - { mtu=conf.ipv6_mtu, counters=conf.counters }) + { mtu=conf.internal_interface.mtu, counters=conf.counters }) config.app(c, "ndp", ipv6_apps.NDP, - { src_ipv6 = conf.aftr_ipv6_ip, src_eth = conf.aftr_mac_b4_side, - dst_eth = conf.next_hop6_mac, dst_ipv6 = conf.next_hop_ipv6_addr }) + { src_ipv6 = conf.internal_interface.ip, + src_eth = conf.internal_interface.mac, + dst_eth = conf.internal_interface.next_hop.mac, + dst_ipv6 = conf.internal_interface.next_hop.ip }) config.app(c, "arp", ipv4_apps.ARP, - { src_ipv4 = conf.aftr_ipv4_ip, src_eth = conf.aftr_mac_inet_side, - dst_eth = conf.inet_mac, dst_ipv4 = conf.next_hop_ipv4_addr}) + { src_ipv4 = convert_ipv4(conf.external_interface.ip), + src_eth = conf.external_interface.mac, + dst_eth = conf.external_interface.next_hop.mac, + dst_ipv4 = convert_ipv4(conf.external_interface.next_hop.ip) }) local preprocessing_apps_v4 = { "reassemblerv4" } local preprocessing_apps_v6 = { "reassemblerv6" } local postprocessing_apps_v4 = { "fragmenterv4" } local postprocessing_apps_v6 = { "fragmenterv6" } - if conf.ipv4_ingress_filter then - config.app(c, "ingress_filterv4", PcapFilter, { filter = conf.ipv4_ingress_filter }) + if conf.external_interface.ingress_filter then + config.app(c, "ingress_filterv4", PcapFilter, + { filter = conf.external_interface.ingress_filter }) append(preprocessing_apps_v4, "ingress_filterv4") end - if conf.ipv6_ingress_filter then - config.app(c, "ingress_filterv6", PcapFilter, { filter = conf.ipv6_ingress_filter }) + if conf.internal_interface.ingress_filter then + config.app(c, "ingress_filterv6", PcapFilter, + { filter = conf.internal_interface.ingress_filter }) append(preprocessing_apps_v6, "ingress_filterv6") end - if conf.ipv4_egress_filter then - config.app(c, "egress_filterv4", PcapFilter, { filter = conf.ipv4_egress_filter }) + if conf.external_interface.egress_filter then + config.app(c, "egress_filterv4", PcapFilter, + { filter = conf.external_interface.egress_filter }) prepend(postprocessing_apps_v4, "egress_filterv4") end - if conf.ipv6_egress_filter then - config.app(c, "egress_filterv6", PcapFilter, { filter = conf.ipv6_egress_filter }) + if conf.internal_interface.egress_filter then + config.app(c, "egress_filterv6", PcapFilter, + { filter = conf.internal_interface.egress_filter }) prepend(postprocessing_apps_v6, "egress_filterv6") end @@ -128,16 +152,16 @@ function load_phy(c, conf, v4_nic_name, v4_nic_pci, v6_nic_name, v6_nic_pci) config.app(c, v4_nic_name, Intel82599, { pciaddr=v4_nic_pci, - vmdq=conf.vlan_tagging, - vlan=conf.vlan_tagging and conf.v4_vlan_tag, + vmdq=conf.external_interface.vlan_tag, + vlan=conf.external_interface.vlan_tag, rxcounter=1, - macaddr=ethernet:ntop(conf.aftr_mac_inet_side)}) + macaddr=ethernet:ntop(conf.external_interface.mac)}) config.app(c, v6_nic_name, Intel82599, { pciaddr=v6_nic_pci, - vmdq=conf.vlan_tagging, - vlan=conf.vlan_tagging and conf.v6_vlan_tag, + vmdq=conf.internal_interface.vlan_tag, + vlan=conf.internal_interface.vlan_tag, rxcounter=1, - macaddr = ethernet:ntop(conf.aftr_mac_b4_side)}) + macaddr = ethernet:ntop(conf.internal_interface.mac)}) link_source(c, v4_nic_name..'.tx', v6_nic_name..'.tx') link_sink(c, v4_nic_name..'.rx', v6_nic_name..'.rx') @@ -151,9 +175,9 @@ function load_on_a_stick(c, conf, args) if v4v6 then config.app(c, 'nic', Intel82599, { pciaddr = pciaddr, - vmdq = conf.vlan_tagging, - vlan = conf.vlan_tagging and conf.v4_vlan_tag, - macaddr = ethernet:ntop(conf.aftr_mac_inet_side)}) + vmdq=conf.external_interface.vlan_tag, + vlan=conf.external_interface.vlan_tag, + macaddr = ethernet:ntop(conf.external_interface.mac)}) if mirror then local Tap = require("apps.tap.tap").Tap local ifname = mirror @@ -173,14 +197,14 @@ function load_on_a_stick(c, conf, args) else config.app(c, v4_nic_name, Intel82599, { pciaddr = pciaddr, - vmdq = conf.vlan_tagging, - vlan = conf.vlan_tagging and conf.v4_vlan_tag, - macaddr = ethernet:ntop(conf.aftr_mac_inet_side)}) + vmdq=conf.external_interface.vlan_tag, + vlan=conf.external_interface.vlan_tag, + macaddr = ethernet:ntop(conf.external_interface.mac)}) config.app(c, v6_nic_name, Intel82599, { pciaddr = pciaddr, - vmdq = conf.vlan_tagging, - vlan = conf.vlan_tagging and conf.v6_vlan_tag, - macaddr = ethernet:ntop(conf.aftr_mac_b4_side)}) + vmdq=conf.internal_interface.vlan_tag, + vlan=conf.internal_interface.vlan_tag, + macaddr = ethernet:ntop(conf.internal_interface.mac)}) link_source(c, v4_nic_name..'.tx', v6_nic_name..'.tx') link_sink(c, v4_nic_name..'.rx', v6_nic_name..'.rx') @@ -192,12 +216,12 @@ function load_virt(c, conf, v4_nic_name, v4_nic_pci, v6_nic_name, v6_nic_pci) config.app(c, v4_nic_name, VirtioNet, { pciaddr=v4_nic_pci, - vlan=conf.vlan_tagging and conf.v4_vlan_tag, - macaddr=ethernet:ntop(conf.aftr_mac_inet_side)}) + vlan=conf.external_interface.vlan_tag, + macaddr=ethernet:ntop(conf.external_interface.mac)}) config.app(c, v6_nic_name, VirtioNet, { pciaddr=v6_nic_pci, - vlan=conf.vlan_tagging and conf.v6_vlan_tag, - macaddr = ethernet:ntop(conf.aftr_mac_b4_side)}) + vlan=conf.internal_interface.vlan_tag, + macaddr = ethernet:ntop(conf.internal_interface.mac)}) link_source(c, v4_nic_name..'.tx', v6_nic_name..'.tx') link_sink(c, v4_nic_name..'.rx', v6_nic_name..'.rx') @@ -210,9 +234,13 @@ function load_bench(c, conf, v4_pcap, v6_pcap, v4_sink, v6_sink) config.app(c, "capturev6", pcap.PcapReader, v6_pcap) config.app(c, "repeaterv4", basic_apps.Repeater) config.app(c, "repeaterv6", basic_apps.Repeater) - if conf.vlan_tagging then - config.app(c, "untagv4", vlan.Untagger, { tag=conf.v4_vlan_tag }) - config.app(c, "untagv6", vlan.Untagger, { tag=conf.v6_vlan_tag }) + if conf.external_interface.vlan_tag then + config.app(c, "untagv4", vlan.Untagger, + { tag=conf.external_interface.vlan_tag }) + end + if conf.internal_interface.vlan_tag then + config.app(c, "untagv6", vlan.Untagger, + { tag=conf.internal_interface.vlan_tag }) end config.app(c, v4_sink, basic_apps.Sink) config.app(c, v6_sink, basic_apps.Sink) @@ -220,13 +248,16 @@ function load_bench(c, conf, v4_pcap, v6_pcap, v4_sink, v6_sink) config.link(c, "capturev4.output -> repeaterv4.input") config.link(c, "capturev6.output -> repeaterv6.input") - if conf.vlan_tagging then - config.link(c, "repeaterv4.output -> untagv4.input") - config.link(c, "repeaterv6.output -> untagv6.input") - link_source(c, 'untagv4.output', 'untagv6.output') - else - link_source(c, 'repeaterv4.output', 'repeaterv6.output') + local v4_src, v6_src = 'repeaterv4.output', 'repeaterv6.output' + if conf.external_interface.vlan_tag then + config.link(c, v4_src.." -> untagv4.input") + v4_src = "untagv4.output" + end + if conf.internal_interface.vlan_tag then + config.link(c, v6_src.." -> untagv6.input") + v6_src = "untagv6.output" end + link_source(c, v4_src, v6_src) link_sink(c, v4_sink..'.input', v6_sink..'.input') end @@ -237,11 +268,17 @@ function load_check_on_a_stick (c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6 config.app(c, "capturev6", pcap.PcapReader, inv6_pcap) config.app(c, "output_filev4", pcap.PcapWriter, outv4_pcap) config.app(c, "output_filev6", pcap.PcapWriter, outv6_pcap) - if conf.vlan_tagging then - config.app(c, "untagv4", vlan.Untagger, { tag=conf.v4_vlan_tag }) - config.app(c, "untagv6", vlan.Untagger, { tag=conf.v6_vlan_tag }) - config.app(c, "tagv4", vlan.Tagger, { tag=conf.v4_vlan_tag }) - config.app(c, "tagv6", vlan.Tagger, { tag=conf.v6_vlan_tag }) + if conf.external_interface.vlan_tag then + config.app(c, "untagv4", vlan.Untagger, + { tag=conf.external_interface.vlan_tag }) + config.app(c, "tagv4", vlan.Tagger, + { tag=conf.external_interface.vlan_tag }) + end + if conf.internal_interface.vlan_tag then + config.app(c, "untagv6", vlan.Untagger, + { tag=conf.internal_interface.vlan_tag }) + config.app(c, "tagv6", vlan.Tagger, + { tag=conf.internal_interface.vlan_tag }) end config.app(c, 'v4v6', V4V6) @@ -251,7 +288,7 @@ function load_check_on_a_stick (c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6 local sources = { "v4v6.v4", "v4v6.v6" } local sinks = { "v4v6.v4", "v4v6.v6" } - if conf.vlan_tagging then + if conf.external_interface.vlan_tag then config.link(c, "capturev4.output -> untagv4.input") config.link(c, "capturev6.output -> untagv6.input") config.link(c, "untagv4.output -> join.in1") @@ -282,17 +319,23 @@ function load_check(c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap) config.app(c, "capturev6", pcap.PcapReader, inv6_pcap) config.app(c, "output_filev4", pcap.PcapWriter, outv4_pcap) config.app(c, "output_filev6", pcap.PcapWriter, outv6_pcap) - if conf.vlan_tagging then - config.app(c, "untagv4", vlan.Untagger, { tag=conf.v4_vlan_tag }) - config.app(c, "untagv6", vlan.Untagger, { tag=conf.v6_vlan_tag }) - config.app(c, "tagv4", vlan.Tagger, { tag=conf.v4_vlan_tag }) - config.app(c, "tagv6", vlan.Tagger, { tag=conf.v6_vlan_tag }) + if conf.external_interface.vlan_tag then + config.app(c, "untagv4", vlan.Untagger, + { tag=conf.external_interface.vlan_tag }) + config.app(c, "tagv4", vlan.Tagger, + { tag=conf.external_interface.vlan_tag }) + end + if conf.internal_interface.vlan_tag then + config.app(c, "untagv6", vlan.Untagger, + { tag=conf.internal_interface.vlan_tag }) + config.app(c, "tagv6", vlan.Tagger, + { tag=conf.internal_interface.vlan_tag }) end local sources = { "capturev4.output", "capturev6.output" } local sinks = { "output_filev4.input", "output_filev6.input" } - if conf.vlan_tagging then + if conf.external_interface.vlan_tag then sources = { "untagv4.output", "untagv6.output" } sinks = { "tagv4.input", "tagv6.input" } @@ -314,11 +357,17 @@ function load_soak_test(c, conf, inv4_pcap, inv6_pcap) config.app(c, "loop_v4", basic_apps.Repeater) config.app(c, "loop_v6", basic_apps.Repeater) config.app(c, "sink", basic_apps.Sink) - if conf.vlan_tagging then - config.app(c, "untagv4", vlan.Untagger, { tag=conf.v4_vlan_tag }) - config.app(c, "untagv6", vlan.Untagger, { tag=conf.v6_vlan_tag }) - config.app(c, "tagv4", vlan.Tagger, { tag=conf.v4_vlan_tag }) - config.app(c, "tagv6", vlan.Tagger, { tag=conf.v6_vlan_tag }) + if conf.external_interface.vlan_tag then + config.app(c, "untagv4", vlan.Untagger, + { tag=conf.external_interface.vlan_tag }) + config.app(c, "tagv4", vlan.Tagger, + { tag=conf.external_interface.vlan_tag }) + end + if conf.internal_interface.vlan_tag then + config.app(c, "untagv6", vlan.Untagger, + { tag=conf.internal_interface.vlan_tag }) + config.app(c, "tagv6", vlan.Tagger, + { tag=conf.internal_interface.vlan_tag }) end local sources = { "loop_v4.output", "loop_v6.output" } @@ -327,7 +376,7 @@ function load_soak_test(c, conf, inv4_pcap, inv6_pcap) config.link(c, "capturev4.output -> loop_v4.input") config.link(c, "capturev6.output -> loop_v6.input") - if conf.vlan_tagging then + if conf.external_interface.vlan_tag then sources = { "untagv4.output", "untagv6.output" } sinks = { "tagv4.input", "tagv6.input" } @@ -349,11 +398,17 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) config.app(c, "loop_v4", basic_apps.Repeater) config.app(c, "loop_v6", basic_apps.Repeater) config.app(c, "sink", basic_apps.Sink) - if conf.vlan_tagging then - config.app(c, "untagv4", vlan.Untagger, { tag=conf.v4_vlan_tag }) - config.app(c, "untagv6", vlan.Untagger, { tag=conf.v6_vlan_tag }) - config.app(c, "tagv4", vlan.Tagger, { tag=conf.v4_vlan_tag }) - config.app(c, "tagv6", vlan.Tagger, { tag=conf.v6_vlan_tag }) + if conf.external_interface.vlan_tag then + config.app(c, "untagv4", vlan.Untagger, + { tag=conf.external_interface.vlan_tag }) + config.app(c, "tagv4", vlan.Tagger, + { tag=conf.external_interface.vlan_tag }) + end + if conf.internal_interface.vlan_tag then + config.app(c, "untagv6", vlan.Untagger, + { tag=conf.internal_interface.vlan_tag }) + config.app(c, "tagv6", vlan.Tagger, + { tag=conf.internal_interface.vlan_tag }) end config.app(c, 'v4v6', V4V6) @@ -366,7 +421,7 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) config.link(c, "capturev4.output -> loop_v4.input") config.link(c, "capturev6.output -> loop_v6.input") - if conf.vlan_tagging then + if conf.external_interface.vlan_tag then config.link(c, "loop_v4.output -> untagv4.input") config.link(c, "loop_v6.output -> untagv6.input") config.link(c, "untagv4.output -> join.in1") diff --git a/src/program/snabbvmx/lwaftr/lwaftr.lua b/src/program/snabbvmx/lwaftr/lwaftr.lua index 1e0556f220..24f7094f1f 100644 --- a/src/program/snabbvmx/lwaftr/lwaftr.lua +++ b/src/program/snabbvmx/lwaftr/lwaftr.lua @@ -76,11 +76,12 @@ local function effective_vlan (conf, lwconf) if conf.settings and conf.settings.vlan then return conf.settings.vlan end - if lwconf.vlan_tagging then - if lwconf.v4_vlan_tag == lwconf.v6_vlan_tag then - return lwconf.v4_vlan_tag + if lwconf.external_interface.vlan_tag then + if lwconf.external_interface.vlan_tag == lwconf.internal_interface.vlan_tag then + return lwconf.external_interface.vlan_tag end - return {v4_vlan_tag = lwconf.v4_vlan_tag, v6_vlan_tag = lwconf.v6_vlan_tag} + return {v4_vlan_tag = lwconf.external_interface.vlan_tag, + v6_vlan_tag = lwconf.internal_interface.vlan_tag} end return false end @@ -88,8 +89,7 @@ end function run(args) local opts, conf_file, id, pci, mac, sock_path, mirror_id = parse_args(args) - local conf = {} - local lwconf = {} + local conf, lwconf local ring_buffer_size = 2048 local ingress_drop_action = "flush" @@ -107,12 +107,13 @@ function run(args) fatal(("lwAFTR conf file '%s' not found"):format(conf.lwaftr)) end lwconf = require('apps.lwaftr.conf').load_legacy_lwaftr_config(conf.lwaftr) - lwconf.ipv6_mtu = lwconf.ipv6_mtu or 1500 - lwconf.ipv4_mtu = lwconf.ipv4_mtu or 1460 + -- If one interface has vlan tags, the other one should as well. + assert((not lwconf.external_interface.vlan_tag) == + (not lwconf.internal_interface.vlan_tag)) else - print(("Interface '%s' set to passthru mode"):format(id)) + print(("Interface '%s' set to passthrough mode."):format(id)) ring_buffer_size = 1024 - conf.settings = {} + conf = {settings = {}} end if conf.settings then @@ -140,19 +141,20 @@ function run(args) lwaftr_id.value = id end - local vlan = effective_vlan(conf, lwconf) - + local vlan = false local mtu = DEFAULT_MTU - if lwconf.ipv6_mtu then - mtu = lwconf.ipv6_mtu - end - if lwconf.ipv4_mtu and lwconf.ipv4_mtu > lwconf.ipv6_mtu then - mtu = lwconf.ipv4_mtu - end - mtu = mtu + constants.ethernet_header_size - if lwconf.vlan_tagging then - mtu = mtu + 4 + if lwconf then + vlan = effective_vlan(conf, lwconf) + mtu = lwconf.internal_interface.mtu + if lwconf.external_interface.mtu > mtu then + mtu = lwconf.external_interface.mtu + end + mtu = mtu + constants.ethernet_header_size + if lwconf.external_interface.vlan_tag then + mtu = mtu + 4 + end end + conf.interface = { mac_address = mac, pci = pci, @@ -163,7 +165,11 @@ function run(args) } local c = config.new() - setup.lwaftr_app(c, conf, lwconf, sock_path) + if lwconf then + setup.lwaftr_app(c, conf, lwconf, sock_path) + else + setup.passthrough(c, conf, sock_path) + end engine.configure(c) if opts.verbosity >= 2 then diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index aab3f54731..b29770702c 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -49,8 +49,8 @@ local function load_virt (c, nic_id, lwconf, interface) print(("%s ether %s"):format(nic_id, interface.mac_address)) local v4_nic_name, v6_nic_name = nic_id..'_v4', nic_id..'v6' - local v4_mtu = lwconf.ipv4_mtu + constants.ethernet_header_size - if lwconf.vlan_tagging and lwconf.v4_vlan_tag then + local v4_mtu = lwconf.external_interface.mtu + constants.ethernet_header_size + if lwconf.external_interface.vlan_tag then v4_mtu = v4_mtu + 4 end print(("Setting %s interface MTU to %d"):format(v4_nic_name, v4_mtu)) @@ -58,10 +58,10 @@ local function load_virt (c, nic_id, lwconf, interface) pciaddr = interface.pci, vmdq = interface.vlan and true, vlan = interface.vlan and interface.vlan.v4_vlan_tag, - macaddr = ethernet:ntop(lwconf.aftr_mac_inet_side), + macaddr = ethernet:ntop(lwconf.external_interface.mac), mtu = v4_mtu }) - local v6_mtu = lwconf.ipv6_mtu + constants.ethernet_header_size - if lwconf.vlan_tagging and lwconf.v6_vlan_tag then + local v6_mtu = lwconf.internal_interface.mtu + constants.ethernet_header_size + if lwconf.internal_interface.vlan_tag then v6_mtu = v6_mtu + 4 end print(("Setting %s interface MTU to %d"):format(v6_nic_name, v6_mtu)) @@ -69,7 +69,7 @@ local function load_virt (c, nic_id, lwconf, interface) pciaddr = interface.pci, vmdq = interface.vlan and true, vlan = interface.vlan and interface.vlan.v6_vlan_tag, - macaddr = ethernet:ntop(lwconf.aftr_mac_b4_side), + macaddr = ethernet:ntop(lwconf.internal_interface.mac), mtu = v6_mtu}) return v4_nic_name, v6_nic_name @@ -117,19 +117,15 @@ local function load_phy (c, nic_id, interface) end local function requires_splitter (lwconf) - if not lwconf.vlan_tagging then return true end - return lwconf.v4_vlan_tag == lwconf.v6_vlan_tag + if not lwconf.internal_interface.vlan_tag then return true end + return lwconf.internal_interface.vlan_tag == lwconf.external_interface.vlan_tag end function lwaftr_app(c, conf, lwconf, sock_path) assert(type(conf) == 'table') assert(type(lwconf) == 'table') - if lwconf.binding_table then - conf.preloaded_binding_table = bt.load(lwconf.binding_table) - end - - print(("Hairpinning: %s"):format(yesno(lwconf.hairpinning))) + print(("Hairpinning: %s"):format(yesno(lwconf.internal_interface.hairpinning))) local counters = lwcounter.init_counters() local virt_id = "vm_" .. conf.interface.id @@ -172,11 +168,13 @@ function lwaftr_app(c, conf, lwconf, sock_path) print(("IPv6 fragmentation and reassembly: %s"):format(yesno( conf.ipv6_interface.fragmentation))) if conf.ipv6_interface.fragmentation then - local mtu = conf.ipv6_interface.mtu or lwconf.ipv6_mtu + local mtu = conf.ipv6_interface.mtu or lwconf.internal_interface.mtu config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { counters = counters, - max_ipv6_reassembly_packets = lwconf.max_ipv6_reassembly_packets, - max_fragments_per_reassembly_packet = lwconf.max_fragments_per_reassembly_packet, + max_ipv6_reassembly_packets = + lwconf.internal_interface.reassembly.max_packets, + max_fragments_per_reassembly_packet = + lwconf.internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, { counters = counters, @@ -207,11 +205,13 @@ function lwaftr_app(c, conf, lwconf, sock_path) print(("IPv4 fragmentation and reassembly: %s"):format(yesno( conf.ipv4_interface.fragmentation))) if conf.ipv4_interface.fragmentation then - local mtu = conf.ipv4_interface.mtu or lwconf.ipv4_mtu + local mtu = conf.ipv4_interface.mtu or lwconf.external_interface.mtu config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { counters = counters, - max_ipv4_reassembly_packets = lwconf.max_ipv4_reassembly_packets, - max_fragments_per_reassembly_packet = lwconf.max_fragments_per_reassembly_packet, + max_ipv4_reassembly_packets = + lwconf.external_interface.reassembly.max_packets, + max_fragments_per_reassembly_packet = + lwconf.external_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, { counters = counters, @@ -288,6 +288,29 @@ function lwaftr_app(c, conf, lwconf, sock_path) end end +function passthrough(c, conf, sock_path) + assert(type(conf) == 'table') + + io.write("lwAFTR service: disabled ") + print("(either empty binding_table or v6 or v4 interface config missing)") + + local virt_id = "vm_" .. conf.interface.id + local phy_id = "nic_" .. conf.interface.id + local chain_input, chain_output = load_phy(c, phy_id, conf.interface) + + if sock_path then + local socket_path = sock_path:format(conf.interface.id) + config.app(c, virt_id, VhostUser, { socket_path = socket_path }) + config.link(c, virt_id .. ".tx -> " .. chain_input) + config.link(c, chain_output .. " -> " .. virt_id .. ".rx") + else + config.app(c, "DummyVhost", basic_apps.Sink) + config.link(c, "DummyVhost" .. ".tx -> " .. chain_input) + config.link(c, chain_output .. " -> " .. "DummyVhost" .. ".rx") + print("Running without VM (no vHostUser sock_path set)") + end +end + local function load_conf (conf_filename) local function load_lwaftr_config (conf, conf_filename) local filename = conf.lwaftr @@ -309,11 +332,13 @@ local function lwaftr_app_check (c, conf, lwconf, sources, sinks) if conf.ipv6_interface then if conf.ipv6_interface.fragmentation then - local mtu = conf.ipv6_interface.mtu or lwconf.ipv6_mtu + local mtu = conf.ipv6_interface.mtu or lwconf.internal_interface.mtu config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { counters = counters, - max_ipv6_reassembly_packets = lwconf.max_ipv6_reassembly_packets, - max_fragments_per_reassembly_packet = lwconf.max_fragments_per_reassembly_packet, + max_ipv6_reassembly_packets = + lwconf.internal_interface.reassembly.max_packets, + max_fragments_per_reassembly_packet = + lwconf.internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, { counters = counters, @@ -339,11 +364,13 @@ local function lwaftr_app_check (c, conf, lwconf, sources, sinks) if conf.ipv4_interface then if conf.ipv4_interface.fragmentation then - local mtu = conf.ipv4_interface.mtu or lwconf.ipv4_mtu + local mtu = conf.ipv4_interface.mtu or lwconf.external_interface.mtu config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { counters = counters, - max_ipv4_reassembly_packets = lwconf.max_ipv4_reassembly_packets, - max_fragments_per_reassembly_packet = lwconf.max_fragments_per_reassembly_packet, + max_ipv4_reassembly_packets = + lwconf.external_interface.reassembly.max_packets, + max_fragments_per_reassembly_packet = + lwconf.external_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, { counters = counters, From 79224b15e17cd2048bb88af2599fc45b61738230 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 8 Nov 2016 17:54:40 +0000 Subject: [PATCH 154/631] Fix snabbvmx, hopefully! --- src/program/snabbvmx/lwaftr/setup.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index b29770702c..818f244073 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -320,7 +320,7 @@ local function load_conf (conf_filename) return require("apps.lwaftr.conf").load_legacy_lwaftr_config(filename) end local conf = dofile(conf_filename) - return conf, load_legacy_lwaftr_config(conf, conf_filename) + return conf, load_lwaftr_config(conf, conf_filename) end local function lwaftr_app_check (c, conf, lwconf, sources, sinks) From 5aec3d9202b5de867c60d9514be40c3afff423d4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 09:53:11 +0100 Subject: [PATCH 155/631] Trim unused code. --- src/apps/lwaftr/binding_table.lua | 8 +------- .../migrate_configuration/migrate_configuration.lua | 7 ------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index 802c3e5180..fbc02390d7 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -73,7 +73,7 @@ local ctable = require("lib.ctable") local cltable = require("lib.cltable") local util = require("lib.yang.util") -local band, bor, bxor, lshift, rshift = bit.band, bit.bor, bit.bxor, bit.lshift, bit.rshift +local band, lshift, rshift = bit.band, bit.lshift, bit.rshift local psid_map_value_t = ffi.typeof[[ struct { uint16_t psid_length; uint16_t shift; } @@ -305,7 +305,6 @@ local function parse_br_addresses(parser) parser:skip_whitespace() if parser:check(',') then parser:skip_whitespace() end end - local ptr = ffi.new(ffi.typeof('$[?]', br_address_t), #addresses) local ret = util.ffi_array(ffi.new(ffi.typeof('$[?]', br_address_t), #addresses), br_address_t, #addresses) @@ -380,11 +379,6 @@ function load_source(text_stream) return parse_binding_table(Parser.new(text_stream)) end -verbose = os.getenv('SNABB_LWAFTR_VERBOSE') or true -local function log(msg, ...) - if verbose then print(msg:format(...)) end -end - function load_legacy(file) local source = stream.open_input_byte_stream(file) return load_source(source:as_text_stream()) diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 6bdc5edcbc..3a3b71ef7b 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -1,14 +1,8 @@ module(..., package.seeall) local lib = require('core.lib') -local ffi = require('ffi') -local util = require('lib.yang.util') -local ipv4 = require('lib.protocol.ipv4') -local ctable = require('lib.ctable') -local binding_table = require("apps.lwaftr.binding_table") local conf = require('apps.lwaftr.conf') local load_legacy_lwaftr_config = conf.load_legacy_lwaftr_config -local ffi_array = require('lib.yang.util').ffi_array local yang = require('lib.yang.yang') local function show_usage(code) @@ -25,7 +19,6 @@ local function parse_args(args) end function run(args) - binding_table.verbose = false local conf_file = parse_args(args) local conf = load_legacy_lwaftr_config(conf_file) yang.print_data_for_schema_by_name('snabb-softwire-v1', conf, io.stdout) From 2dedf6aa4e47f12bc5fd632c4fe9bfc5c8913b3e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 10:39:00 +0100 Subject: [PATCH 156/631] Conf passed to lwaftr app must be object Loading conf from a file hasn't worked for a while. --- src/apps/lwaftr/lwaftr.lua | 4 ---- src/program/lwaftr/run_nohw/run_nohw.lua | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 14ce7f8200..8fa0295190 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -4,7 +4,6 @@ local bt = require("apps.lwaftr.binding_table") local constants = require("apps.lwaftr.constants") local dump = require('apps.lwaftr.dump') local icmp = require("apps.lwaftr.icmp") -local lwconf = require("apps.lwaftr.conf") local lwdebug = require("apps.lwaftr.lwdebug") local lwheader = require("apps.lwaftr.lwheader") local lwutil = require("apps.lwaftr.lwutil") @@ -241,9 +240,6 @@ end LwAftr = {} function LwAftr:new(conf) - if type(conf) == 'string' then - conf = lwconf.load_legacy_lwaftr_config(conf) - end if conf.debug then debug = true end local o = setmetatable({}, {__index=LwAftr}) o.conf = conf diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index 51ab101adf..a4bf60f52d 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -55,10 +55,11 @@ end function run(parameters) local verbosity, conf_file, b4_if, inet_if, bench_file = parse_args(parameters) + local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local c = config.new() -- AFTR - config.app(c, "aftr", LwAftr, conf_file) + config.app(c, "aftr", LwAftr, conf) -- B4 side interface config.app(c, "b4if", RawSocket, b4_if) From 957a45add382519442ce6171109c92d94978021b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 11:43:23 +0100 Subject: [PATCH 157/631] Fix yang source parser bug for file streams --- src/lib/yang/stream.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/stream.lua b/src/lib/yang/stream.lua index 2800b7e3a7..fd5b0f688b 100644 --- a/src/lib/yang/stream.lua +++ b/src/lib/yang/stream.lua @@ -143,7 +143,8 @@ function open_input_byte_stream(filename) return ffi.string(ret:read(1), 1) end function ret:read_string() - return ffi.string(ret:read(size - pos), size - pos) + local count = size - pos + return ffi.string(ret:read(count), count) end function ret:as_text_stream(len) local end_pos = size From 08b189d016e40c0bb4dcb4b5947299890c99888f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 11:44:00 +0100 Subject: [PATCH 158/631] Implement configuration parsing for all yang table types --- src/lib/yang/data.lua | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 12df6916f6..e278c325d1 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -238,6 +238,22 @@ local function string_keyed_table_builder(string_key) return builder end +local function cltable_builder(key_t) + local res = cltable.new({ key_type=key_t }) + local builder = {} + function builder:add(key, value) res[key] = value end + function builder:finish() return res end + return builder +end + +local function ltable_builder() + local res = {} + local builder = {} + function builder:add(key, value) res[key] = value end + function builder:finish() return res end + return builder +end + local function table_parser(keyword, keys, values, string_key, key_ctype, value_ctype) local members = {} @@ -251,13 +267,10 @@ local function table_parser(keyword, keys, values, string_key, key_ctype, function init() return ctable_builder(key_t, value_t) end elseif string_key then function init() return string_keyed_table_builder(string_key) end + elseif key_t then + function init() return cltable_builder(key_t) end else - -- TODO: here we should implement a cktable if key_t is non-nil. - -- Probably we should error if the key is a generic Lua table - -- though, as we don't have a good data structure to map generic - -- Lua tables to Lua tables. For the moment, fall back to the old - -- assoc implementation. - error('List with non-FFI, non-string key unimplemented') + function init() return ltable_builder() end end local function parse1(node) assert_compound(node, keyword) From 03ad2daa252fc2991af2a23872872b0ff4d8ac76 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Wed, 9 Nov 2016 11:45:56 +0100 Subject: [PATCH 159/631] Added documentation on how to run on-a-stick mode --- src/program/lwaftr/doc/README.running.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/doc/README.running.md b/src/program/lwaftr/doc/README.running.md index 40417adbf3..9224295ac9 100644 --- a/src/program/lwaftr/doc/README.running.md +++ b/src/program/lwaftr/doc/README.running.md @@ -27,7 +27,7 @@ with the appropriate prefix (`0000:`, in this example). Note: Compile Snabb (see [README.build.md](README.build.md)) before attempting the following. -## Running a load generator and the lwaftr +## Running a load generator and the lwaftr (2 lwaftr NICs) To run a load generator and an `lwaftr`, you will need four interfaces. The following example assumes that `01:00.0` is cabled to @@ -62,6 +62,19 @@ ramping up from 0 Gbps to 10 Gbps (by default). At each step it measures the return traffic from the lwAFTR, and prints out all this information to the console. The load generator stops when the test is done. +## Running the lwaftr on one NIC ('on a stick') + +To more efficiently use network bandwidth, it makes sense to run the lwaftr on +just one NIC, because traffic is not symmetric. To do this, use --on-a-stick, +and specify only one PCI address: + +``` +$ sudo ./snabb lwaftr run \ + --conf program/lwaftr/tests/data/icmp_on_fail.conf \ + --on-a-stick 0000:02:00.1 +``` + + ## The packetblaster Another way of generating load is via the `packetblaster lwaftr` command, From 278807162bdf58a1c56db60581a0ed224d3dd475 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 11:54:44 +0100 Subject: [PATCH 160/631] Fix integer parsing bug in yang --- src/lib/yang/util.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index a181548638..7809f2fd55 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -38,7 +38,12 @@ function tointeger(str, what, min, max) end if max then check(res <= max) end -- Only return Lua numbers for values within int32 + uint32 range. - if -0x8000000 <= res and res <= 0xffffffff then return tonumber(res) end + -- The 0 <= res check is needed because res might be a uint64, in + -- which case comparing to a negative Lua number will cast that Lua + -- number to a uint64 :-(( + if (0 <= res or -0x8000000 <= res) and res <= 0xffffffff then + return tonumber(res) + end return res end @@ -80,6 +85,7 @@ function selftest() assert(tointeger('0') == 0) assert(tointeger('-0') == 0) assert(tointeger('10') == 10) + assert(tostring(tointeger('10')) == '10') assert(tointeger('-10') == -10) assert(tointeger('010') == 8) assert(tointeger('-010') == -8) From 0e316539695274c9b5bf227c1780e8e967b408d3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 11:59:22 +0100 Subject: [PATCH 161/631] Fix serialization bug in yang code --- src/lib/yang/binary.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 0bef3b70b6..9364b3b6a5 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -139,7 +139,6 @@ local function data_emitter(production) end function handlers.array(production) if production.ctype then - local emit_value = value_emitter(production.ctype) return function(data, stream) stream:write_stringref('carray') stream:write_stringref(production.ctype) @@ -183,7 +182,7 @@ local function data_emitter(production) local emit_value = visit1({type='struct', members=production.values}) return function(data, stream) stream:write_stringref('cltable') - emit_keys(data.keys) + emit_keys(data.keys, stream) stream:write_uint32(#data.values) for i=1,#data.values do emit_value(data.values[i], stream) end end From c6a0e8513222f33e1877a6326e010bcad33a7fe9 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 12:05:28 +0100 Subject: [PATCH 162/631] Fix cltable binary loading in yang --- src/lib/yang/binary.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 9364b3b6a5..d293a6069b 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -9,6 +9,7 @@ local value = require("lib.yang.value") local stream = require("lib.yang.stream") local data = require('lib.yang.data') local ctable = require('lib.ctable') +local cltable = require('lib.cltable') local MAGIC = "yangconf" local VERSION = 0x00001000 From 68e325ee3035ea1f4f654846834834427c62f170 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 12:05:44 +0100 Subject: [PATCH 163/631] Binary configuration saving strips off .conf --- src/lib/yang/yang.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 7645c80aa9..172a1669d4 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -82,7 +82,7 @@ function load_configuration(filename, opts) -- If the file doesn't have the magic, assume it's a source file. -- First, see if we compiled it previously and saved a compiled file -- in a well-known place. - local compiled_filename = filename:gsub("%.txt$", "")..'.o' + local compiled_filename = filename:gsub("%.conf$", "")..'.o' local source_mtime = {sec=source.mtime_sec, nsec=source.mtime_nsec} local compiled_stream = maybe(stream.open_input_byte_stream, compiled_filename) From 528866acb1a2ab29999923d31c48f970ca824f88 Mon Sep 17 00:00:00 2001 From: Christian Graf Date: Wed, 9 Nov 2016 12:22:21 +0100 Subject: [PATCH 164/631] Changes to be committed: modified: README.troubleshooting.md --- .../snabbvmx/doc/README.troubleshooting.md | 455 ++++++++++++++++-- 1 file changed, 404 insertions(+), 51 deletions(-) diff --git a/src/program/snabbvmx/doc/README.troubleshooting.md b/src/program/snabbvmx/doc/README.troubleshooting.md index 27de9682d1..69ae0cfe38 100644 --- a/src/program/snabbvmx/doc/README.troubleshooting.md +++ b/src/program/snabbvmx/doc/README.troubleshooting.md @@ -164,9 +164,17 @@ would like to monitor. For testing purposes, set `lwaftr monitor` to `all`. ## Tests overview -Tests are useful to detect bugs in the `snabbvmx` and double-check that +Tests are useful to detect bugs in the `lwaftr` or `snabbvmx` and double-check that everything is working as expected. +**preparation** +- have tcpdump, snabb or vmxlwaftr installed. +- should work on xeon or i7; will fail on Celeron chips +- hugepages : Ensure hugepages is set in sysctl +"vm.nr_hugepages = 5" in sysctl.conf +If using vmxlwaftr, then this is done by the scripts automatically + +**Test types** SnabbVMX features three types of tests: * Lua selftests: Unit tests for several modules @@ -186,6 +194,9 @@ All these tests are run by Snabb's Continuous Integration subsystem (snabb-bot). The tests can also be run when executing **make test** (only if the Snabb source is available). + +### make tests + ```bash $ sudo make test TEST apps.lwaftr.V4V6 @@ -308,38 +319,50 @@ $ packetblaster replay -D 10 $PCAP_INPUT/v4v6-256.pcap $SNABB_PCI1 NOTE: Currently the test is not working correctly: the returned MAC should be `02:99:99:99:99:99`. -### End-to-end tests -**program/snabbvmx/tests/end-to-end/selftest.sh**: Runs end-to-end tests -(normal and VLAN). +## End-to-end tests - details -The end-to-end tests are a test suite that tests the correctness of the lwAFTR -logic. +**lwaftr vs snabbvmx** +As both lwaftr and snabbvmx provide a different functionality and use different config-files, both the lwaftr and snabbvmx have their dedicated end-to-end tests. -Files in **program/snabbvmx/tests/end-to-end/**: +Although SnabbVMX works on a single interface, `snabbvmx check` requires that +the packet split (IPv4 / IPv6) is already done and provides a split output too. -* **test_env.sh**: Contains the test cases. -* **core-end-to-end.sh**: Runs the test cases using **snabbvmx check**. -* **data/**: Directory containing sample pcap input, expected pcap output, -configuration files and binding tables. -* **end-to-end.sh**: Runs **core-end-to-end.sh** on normal packets. -* **end-to-end-vlan.sh**: Runs **core-end-to-end.sh** on VLAN packets. -* **selftest.sh**: Runs both **end-to-end.sh** and **end-to-end-vlan.sh** +Snabb's lwAFTR includes an end-to-end test suite counterpart. In most cases, +the lwAFTR's correctness will be tested via Snabb's lwAFTR end-to-end tests. +However, since SnabbVMX uses a different configuration file, the network design +that it brings up might be slightly different than Snabb's lwAFTR. For instance, +Snabb's lwAFTR fragmentation is always active, while in SnabbVMX it is an optional +argument, either for IPV4 and IPv6 interfaces. -To run SnabbVMX's end-to-end test suite: +Modifications in the app chain might execute the lwAFTR's data plane in a different +way, bringing up conditions that were not covered by the lwAFTR's data-plane. +For this reason a specific end-to-end test suite was added, covering specifics +needs of SnabbVMX. If a bug is found, its resolution will most likely happen +in Snabb's lwAFTR code, resulting in the addition of a new test to the lwAFTR's +test suite. -```bash -$ sudo ./end-to-end.sh -Testing: IPv6 fragments and fragmentation is off -done -Test passed -All end-to-end lwAFTR tests passed. + +**lwaftr** +``` +cd src/program/lwaftr/tests/end-to-end +``` +**snabbvmx** +``` +cd src/program/snabbvmx/tests/end-to-end ``` -The end-to-end test suite relies on `snabbvmx check` to run each test: +**interactive and scripted tests** +To develop an end-to-end tests, it's recommended to first run it interactively. Once the config-files, pcaps and counters are derived, the test can be added to the scripted tests in test_env.sh. +**interactive** +To run an interactive end-to-end test, either use the snabbvmx or lwaftr app - Have in mind that the test is running with the app specified (lwaftr or snabbvmx) -```bash +- snabb snabbvmx check +- snabb lwaftr check + +**end-to-end interactive usage** +``` $ sudo ./snabb snabbvmx check Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP [COUNTERS.LUA] @@ -347,31 +370,15 @@ Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP Parameters: -- **CONF**: SnabbVMX configuration file. +- **CONF**: SnabbVMX (icmp_snabbvmx-lwaftr-xe.cfg) or lwaftr (icmp_snabbvmx-lwaftr-xe1.conf) configuration file. - **V4-IN.PCAP**: Incoming IPv4 packets (from Internet). - **V6-IN.PCAP**: Incoming IPv6 packets (from b4). - **V4-OUT.PCAP**: Outgoing IPv4 packets (to Internet, decapsulated). - **V6-OUT.PCAP**: Outgoing IPv6 packets (to b4, encapsulated) -- **[COUNTERS.LUA]**: Lua file with counter values. +- **[COUNTERS.LUA]**: Lua file with counter values. Will be regenerated via [-r] parm -Although SnabbVMX works on a single interface, `snabbvmx check` requires that -the packet split is already done and provides a split output too. -Snabb's lwAFTR includes an end-to-end test suite counterpart. In most cases, -the lwAFTR's correctness will be tested via Snabb's lwAFTR end-to-end tests. -However, since SnabbVMX uses a different configuration file, the network design -that it brings up might be slightly different than Snabb's lwAFTR. For instance, -Snabb's lwAFTR fragmentation is always active, while in SnabbVMX it is an optional -argument, either for IPV4 and IPv6 interfaces. - -Modifications in the app chain might execute the lwAFTR's data plane in a different -way, bringing up conditions that were not covered by the lwAFTR's data-plane. -For this reason a specific end-to-end test suite was added, covering specifics -needs of SnabbVMX. If a bug is found, its resolution will most likely happen -in Snabb's lwAFTR code, resulting in the addition of a new test to the lwAFTR's -test suite. - -## How to write a SnabbVMX end-to-end test +## How to run SnabbVMX interactive end-to-end test If you detected an error in the lwAFTR, the first step is to obtain the configuration file that SnabbVMX was using, as well as a copy of lwAFTR's @@ -379,24 +386,370 @@ configuration and binding table. With that information and knowing the error report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get decapsulated, etc), you craft a hand-made packet that meets the testing case. -Now we can check what the lwAFTR produces: +**obtaining the config-files** +To run a test, the following config-files are required: +- the binding-table : binding_table.txt.s +- lwaftr conf : snabbvmx-lwaftr-xe[0-9].conf +- snabbvmx cfg : snabbvmx-lwaftr-xe[0-9].cfg -```bash -sudo ./snabb snabbvmx check -r snabbvmx-lwaftr-xe0.cfg ipv4-pkt.pcap \ - empty.pcap /tmp/outv4.pcap /tmp/outv6.pcap counters.lua +If you are running lwaftr check, then snabbvmx config-file (snabbvmx-lwaftr-xe[0-9].cfg) is not required + +It is fine to copy or manually craft the config-files. +A running snabbvmx can be used as well to copy the config-files from the running container +To gain the used config-files from the running container, either run the collect-support-infos.sh (https://github.com/mwiget/vmxlwaftr/blob/igalia/SUPPORT-INFO.md) +or execute a shell within the dockers container and copy configs and binding-table from the /tmp directory. + +Note: the check application is just using a single interface. If the running container consists of two or more snabb-instances, then just take one of them for when running the check. + +**collect-support-infos.sh** +``` +lab@ubuntu1:~/vmxlwaftr/tests$ ./collect-support-infos.sh lwaftr3-16.2R3 +collecting data in container lwaftr3-16.2R3 ... +tar: Removing leading `/' from member names +tar: stats.xml: Cannot stat: No such file or directory +tar: Removing leading `../' from member names +tar: Exiting with failure status due to previous errors +transferring data from the container to host ... +-rw-r--r-- 1 lab lab 22700552 Nov 8 13:35 support-info-20161108-1335.tgz + +lab@ubuntu1:~/vmxlwaftr/tests/t1$ tar -tvzf support-info-20161108-1335.tgz +-rw-rw-r-- root/root 55 2016-10-31 14:07 VERSION +-rw-r--r-- root/root 1166 2016-11-08 13:35 snabb_xe0.log +-rw-r--r-- root/root 1167 2016-11-08 13:35 snabb_xe1.log +-rw-r--r-- root/root 18 2016-11-04 15:36 mac_xe0 +-rw-r--r-- root/root 18 2016-11-04 15:36 mac_xe1 +-rw-r--r-- root/root 16 2016-11-04 15:36 pci_xe0 +-rw-r--r-- root/root 16 2016-11-04 15:36 pci_xe1 +-rw-r--r-- root/root 86560454 2016-11-08 13:35 binding_table.txt +-rw-r--r-- root/root 7132 2016-11-08 13:35 sysinfo.txt +-rw-r--r-- root/root 139505710 2016-11-04 15:41 binding_table.txt.s +-rw-r--r-- root/root 297 2016-11-04 15:42 snabbvmx-lwaftr-xe0.cfg +-rw-r--r-- root/root 297 2016-11-04 15:42 snabbvmx-lwaftr-xe1.cfg +-rw-r--r-- root/root 377 2016-11-04 15:42 test-snabbvmx-lwaftr-xe0.cfg +-rw-r--r-- root/root 377 2016-11-04 15:43 test-snabbvmx-lwaftr-xe1.cfg +-rw-r--r-- root/root 452 2016-11-04 15:42 snabbvmx-lwaftr-xe0.conf +-rw-r--r-- root/root 377 2016-11-04 15:42 snabbvmx-lwaftr-xe1.conf +-rw-r--r-- root/root 1239 2016-11-08 13:35 config.new +-rw-r--r-- root/root 1699 2016-11-08 13:35 config.new1 +-rw-r--r-- root/root 1239 2016-11-04 15:41 config.old +-rwxr-xr-x root/root 167 2016-11-04 15:42 test_snabb_lwaftr_xe0.sh +-rwxr-xr-x root/root 167 2016-11-04 15:43 test_snabb_lwaftr_xe1.sh +-rwxr-xr-x root/root 204 2016-11-04 15:42 test_snabb_snabbvmx_xe0.sh +-rwxr-xr-x root/root 204 2016-11-04 15:43 test_snabb_snabbvmx_xe1.sh +-rw-r--r-- root/root 33499 2016-11-04 15:36 config_drive/vmm-config.tgz +-rw-r--r-- root/root 3126 2016-11-04 15:36 root/.bashrc +-rwxr-xr-x root/root 2707019 2016-10-31 14:06 usr/local/bin/snabb ``` -The flag `-r` generates a counters file. +**/tmp inside docker container** +The snabbvmx config-files can be derived from the container's shell as well directly +``` +lab@ubuntu1:~/vmxlwaftr/tests/t1$ docker exec -ti lwaftr3-16.2R3 bash +pid 2654's current affinity mask: fffff +pid 2654's new affinity mask: ff3ff +root@8f5d057b8298:/# ls /tmp/ +binding_table.txt config.new mac_xe0 snabb_xe0.log snabbvmx-lwaftr-xe1.cfg test-snabbvmx-lwaftr-xe0.cfg test_snabb_snabbvmx_xe0.sh vhost_features_xe1.socket +binding_table.txt.s config.new1 mac_xe1 snabb_xe1.log snabbvmx-lwaftr-xe1.conf test-snabbvmx-lwaftr-xe1.cfg test_snabb_snabbvmx_xe1.sh vmxhdd.img +binding_table.txt.s.new config.old pci_xe0 snabbvmx-lwaftr-xe0.cfg support-info.tgz test_snabb_lwaftr_xe0.sh vFPC-20160922.img xe0.socket +binding_table.txt.s.o junos-vmx-x86-64-16.1-20160926.0.qcow2 pci_xe1 snabbvmx-lwaftr-xe0.conf sysinfo.txt test_snabb_lwaftr_xe1.sh vhost_features_xe0.socket xe1.socket +``` +Note: press ctrl p ctrl q to exit the containers shell -Check that your output matches what you expect: -```bash -$ tcpdump -qns 0 -ter /tmp/outv4.pcap +**some adoption of config-files is required** +The advantage of snabbvmx is the dynamic next-hop resolution via Junos. When running the the lwaftr or snabbvmx app standalone, then the next-hop resolution via Junos is missing and this must be corrected. +**config as derived from a running vmxlwaftr container** +``` +lab@ubuntu1:~/vmxlwaftr/tests/t1$ cat snabbvmx-lwaftr-xe1.cfg +return { + lwaftr = "snabbvmx-lwaftr-xe1.conf", + settings = { + }, + ipv6_interface = { + ipv6_address = "", + cache_refresh_interval = 1, + fragmentation = false, + }, + ipv4_interface = { + ipv4_address = "192.168.5.2", + cache_refresh_interval = 1, + fragmentation = false, + }, +} +``` + +**Adopted config to use with vmxlwaftr** +Please make sure to change and add the below: +- cache_refresh_interval = 0 (turns off next-hop learning via Junos) +- mac_address (thats the own/self mac-address fo lwaftr) +- next_hop_mac (next-hop mac to send the packets to) + +Note: as seen, the snabbvmx config icmp_snabbvmx-lwaftr-xe1.cfg references the snabb-configuration file icmp_snabbvmx-lwaftr-xe1.conf. + +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat icmp_snabbvmx-lwaftr-xe1.cfg +return { + lwaftr = "icmp_snabbvmx-lwaftr-xe1.conf", + settings = { + }, + ipv6_interface = { + ipv6_address = "", + cache_refresh_interval = 0, <<< set this to 0 + fragmentation = false, + mac_address = "02:cf:69:15:81:01", <<< the lwaftr's own mac address. input pcap match this mac + next_hop_mac = "90:e2:ba:94:2a:bc", <<< the next-hop mac to use for outgoing packets + }, + ipv4_interface = { + ipv4_address = "192.168.5.2", + cache_refresh_interval = 0, <<< set this to 0 + fragmentation = false, + mac_address = "02:cf:69:15:81:01", <<< the lwaftr's own mac address. input pcap match this mac + next_hop_mac = "90:e2:ba:94:2a:bc", <<< the next-hop mac to use for outgoing packets + }, +} +``` + +**The input pcaps** +The check app requires one or two input pcaps. +It is ok to: +- only feed V4-IN.PCAP +- only feed the V6-IN.PCAP +- or feed both V4-IN.PCAP and V6-IN.PCAP + +Note for interactive tests: +When only feeding one pcap, then the other empty pcap must be the empty.pcap in +src/program/snabbvmx/tests/end-to-end/data/empty.pcap + +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ ls empty.pcap +empty.pcap +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ file empty.pcap +empty.pcap: tcpdump capture file (little-endian) - version 2.4 (Ethernet, capture length 65535) +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -r empty.pcap reading from file empty.pcap, link-type EN10MB (Ethernet) ``` -Checking what values are in the counters can give you a hint about whether +**sample icmp-ipv4-in.pcap** +For below sample icmp-ipv4-in.pcap 6 * packets are seen. Only the 1st and 3rd frame is matching the binding-table and shall the encapsulated into lw4o6. The remaining 4 are not matching the binding-table. For those 4 packets the setting of policy_icmpv4_outgoing will define if the lwaftr will generate icmp dest unreachble or just silently discard the packets. +see lwaftr config-file: icmp_snabbvmx-lwaftr-xe1.conf. + +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat cg-binding_table.txt.s +psid_map { + 10.10.0.0 {psid_length=6, shift=10} + 10.10.0.1 {psid_length=6, shift=10} + 10.10.0.10 {psid_length=6, shift=10} +} +br_addresses { + 2a02:587:f700::100, +} +softwires { + { ipv4=10.10.0.0, psid=1, b4=2a02:587:f710::40 } + { ipv4=10.10.0.0, psid=2, b4=2a02:587:f710::41 } + { ipv4=10.10.0.0, psid=3, b4=2a02:587:f710::42 } + { ipv4=10.10.0.0, psid=4, b4=2a02:587:f710::43 } +} + +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -r icmp-ipv4-in.pcap -e +reading from file icmp-ipv4-in.pcap, link-type EN10MB (Ethernet) +11:27:10.976929 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 42: 10.0.1.100.domain > 10.10.0.0.1024: [|domain] +11:27:10.988846 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 42: 10.0.1.100.domain > 10.10.0.0.domain: [|domain] +11:27:11.001278 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 242: 10.0.1.100 > 10.10.0.0: ICMP echo reply, id 1024, seq 0, length 208 +11:27:11.017448 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 242: 10.0.1.100 > 10.10.0.0: ICMP echo reply, id 53, seq 0, length 208 +11:27:11.029226 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 242: 192.168.0.0 > 192.168.1.100: ICMP echo reply, id 1024, seq 0, length 208 +11:27:11.040811 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 42: 192.168.0.0.domain > 192.168.1.100.1024: [|domain] +``` + + + +**running the interactive check** + +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ sudo ~/latest-snabb-binary/snabb/src/snabb snabbvmx check -r ./icmp_snabbvmx-lwaftr-xe1.cfg "./icmp-ipv4-in.pcap" "./empty.pcap" "./outv4/outv4.pcap" "./outv6/icmp-ipv6-out.pcap" icmp.lua +loading compiled binding table from ./cg-binding_table.txt.s.o +compiled binding table ./cg-binding_table.txt.s.o is up to date. +nh_fwd4: cache_refresh_interval set to 0 seconds +nh_fwd4: static next_hop_mac 90:e2:ba:94:2a:bc +nh_fwd6: cache_refresh_interval set to 0 seconds +nh_fwd6: static next_hop_mac 90:e2:ba:94:2a:bc +done +``` + +**results** + +Check that your output matches what you expect: +For given input, the output is matching what we expect: +- 2 translated packets +- an empty outv4.pcap. lwaftr did not generate icmp dst-unreachable, because of the +```bash +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv6/icmp-ipv6-out.pcap +reading from file outv6/icmp-ipv6-out.pcap, link-type EN10MB (Ethernet) +01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::40: IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] +01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::40: IP 10.0.1.100 > 10.10.0.0: ICMP echo reply, id 1024, seq 0, length 208 + +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv4/outv4.pcap +reading from file outv4/outv4.pcap, link-type EN10MB (Ethernet) +``` + +**checking the counters** +The -r flag is set, as such the resulting counters file icmp.lua is generated freshly. The file does match our expectation: +- 2 * IPv6 packets out >> ["out-ipv6-packets"] = 2 +- 4 packets dropped becasue of the policy, as such outv4.pcap is an empty pcap-file +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ grep policy_icmpv4_outgoing icmp_snabbvmx-lwaftr-xe1.conf +policy_icmpv4_outgoing = drop, +``` + +**cat icmp.lua** +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat icmp.lua +return { + ["drop-all-ipv4-iface-bytes"] = 568, + ["drop-all-ipv4-iface-packets"] = 4, + ["drop-no-dest-softwire-ipv4-bytes"] = 568, + ["drop-no-dest-softwire-ipv4-packets"] = 4, + ["drop-out-by-policy-icmpv4-packets"] = 4, + ["in-ipv4-bytes"] = 852, + ["in-ipv4-packets"] = 6, + ["out-ipv6-bytes"] = 364, + ["out-ipv6-packets"] = 2, +} +``` + + +**summary interactive check** + +At this stage the interactive test is finished. +The following is defined: +- lwaftr and snabbvmx configs +- binding-table +- input pcaps + +Further more, the interactive tests produced results, which is: +- expected out V4-OUT.PCAP and V6-OUT.PCAP (which can be empty) +- when using -r switch, then a counters-file got generated as well + +With known input and results, the test can now be added to the scripted end-to-end.sh - to be executed with all other tests to ensure snabbvmx behaves as it should. + +**Note:** +Further more, this procedure can be ideally used to report issues! + +The input-pcaps are Checking what values are in the counters can give you a hint about whether things are working correctly or not. Tip: packets always arrive only in one interface, but the output might be empty for both interfaces, non-empty, and empty or non-empty for both cases. + + +## adding the icmp-test towards the scripted end-to-end tests + +**structure end-to-end tests** +**program/snabbvmx/tests/end-to-end/selftest.sh**: Runs end-to-end tests +(normal and VLAN). + +The end-to-end tests are a test suite that tests the correctness of the lwAFTR +logic. + +Files in **program/snabbvmx/tests/end-to-end/**: + +* **test_env.sh**: Contains the test cases. +* **core-end-to-end.sh**: Runs the test cases using **snabbvmx check**. +* **data/**: Directory containing sample pcap input, expected pcap output, +configuration files and binding tables. +* **end-to-end.sh**: Runs **core-end-to-end.sh** on normal packets. +* **end-to-end-vlan.sh**: Runs **core-end-to-end.sh** on VLAN packets. +* **selftest.sh**: Runs both **end-to-end.sh** and **end-to-end-vlan.sh** + +**Note**: make sure all pcap and config-files are located in the data-drectory +``` +/home/lab/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data +``` +**Note**: make sure all counter-files counters-drectory +``` +/home/lab/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data/counters +``` + +**adding the icmp-test**: + +All tests are defined in the test_env.sh file. +This file has to be edited to include the icmp-test for example. +The check app has two criteria's to decide if the test is successful or not: +- check the counters-file +- check the resulting out.pcap file + +When running the interactive check with the -r option, then a "good" counters-file is created. This counters-file (here icmp.lua) can be referenced in the test_env.sh and if the actual test produced different values as defined in the icmp.lua file, then the test will fail +Another option is to rely on the outgoing pcaps. When running the interactive check, the resulting out.pcap files have been written. The test_env.sh can now run the same test and compare its pcap with the previous stored (./outv4/outv4.pcap , ./outv6/icmp-ipv6-out.pcap) files. If the files differ, then the test fails. + +**pass-criteria based on counters-file** +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ vi test_env.sh +... +TEST_DATA=( + "ICMP Test - pass-critria out.pcap" + "icmp_snabbvmx-lwaftr-xe1.cfg" "icmp-ipv4-in.pcap" "" "" "" + "icmp.lua" +... + "IPv6 fragments and fragmentation is off" + "snabbvmx-lwaftr-xe1.cfg" "" "regressiontest-signedntohl-frags.pcap" "" "" + "drop-all-ipv6-fragments.lua" +) +``` +Running the end-to end.sh based on this configuration file shall not try to compare the out.pcap files. It shall only if the resulting counters are matching the previous defined icmp.lua counters. +If counters do match, then the test passed. + + +**pass-criteria based on pcap-file** +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ vi test_env.sh +... +# Contains an array of test cases. +# +# A test case is a group of 7 data fields, structured as 3 rows: +# - "test_name" +# - "snabbvmx_conf" "v4_in.pcap" "v6_in.pcap" "v4_out.pcap" "v6_out.pcap" +# - "counters" +# +# Notice spaces and new lines are not taken into account. +TEST_DATA=( + "ICMP Test - pass-critria out.pcap" + "icmp_snabbvmx-lwaftr-xe1.cfg" "icmp-ipv4-in.pcap" "empty.pcap" "" "icmp-ipv6-out.pcap" + "" + + "IPv6 fragments and fragmentation is off" + "snabbvmx-lwaftr-xe1.cfg" "" "regressiontest-signedntohl-frags.pcap" "" "" + "drop-all-ipv6-fragments.lua" +) + + +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ sudo ./end-to-end.sh +Testing: ICMP Test - pass-critria out.pcap +loading compiled binding table from data/cg-binding_table.txt.s.o +compiled binding table data/cg-binding_table.txt.s.o is up to date. +nh_fwd4: cache_refresh_interval set to 0 seconds +nh_fwd4: static next_hop_mac 90:e2:ba:94:2a:bc +nh_fwd6: cache_refresh_interval set to 0 seconds +nh_fwd6: static next_hop_mac 90:e2:ba:94:2a:bc +done +Test passed +Testing: IPv6 fragments and fragmentation is off +loading compiled binding table from data/binding_table.txt.s.o +compiled binding table data/binding_table.txt.s.o is up to date. +nh_fwd4: cache_refresh_interval set to 0 seconds +nh_fwd4: static next_hop_mac 90:e2:ba:94:2a:bc +nh_fwd6: cache_refresh_interval set to 0 seconds +nh_fwd6: static next_hop_mac 90:e2:ba:94:2a:bc +done +Test passed +All end-to-end lwAFTR tests passed. +``` + + + + + + + + + + + From c82ee0492fd4a746fe9619b89fcfd69ea9dc9b4d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 12:23:52 +0100 Subject: [PATCH 165/631] Implement yang-syntax string serialization --- src/lib/yang/data.lua | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index e278c325d1..893e4df645 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -345,7 +345,23 @@ end local function encode_yang_string(str) if str:match("^[^%s;{}\"'/]*$") then return str end - error('yang string escaping unimplemented: '..str) + local out = {} + table.insert(out, '"') + for i=1,#str do + local chr = str:sub(i,i) + if chr == '\n' then + table.insert(out, '\\n') + elseif chr == '\t' then + table.insert(out, '\\t') + elseif chr == '"' or chr == '\\' then + table.insert(out, '\\') + table.insert(out, chr) + else + table.insert(out, chr) + end + end + table.insert(out, '"') + return table.concat(out) end local value_printers = {} From 6f4c43e21f089e512e260d085b1d1b79d2905bd4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 12:29:04 +0100 Subject: [PATCH 166/631] Migrate "snabb lwaftr" to yang configurations This also migrates all .conf files that are part of the repository to the new YANG configurations. --- src/program/lwaftr/bench/bench.lua | 2 +- src/program/lwaftr/check/check.lua | 3 +- src/program/lwaftr/soaktest/soaktest.lua | 2 +- .../lwaftr/tests/data/big_mtu_no_icmp.conf | 173 ++++++++++++++-- .../lwaftr/tests/data/icmp_on_fail.conf | 173 ++++++++++++++-- src/program/lwaftr/tests/data/no_hairpin.conf | 173 ++++++++++++++-- src/program/lwaftr/tests/data/no_icmp.conf | 173 ++++++++++++++-- .../lwaftr/tests/data/no_icmp_maxfrags1.conf | 175 +++++++++++++++-- .../data/no_icmp_with_filters_accept.conf | 181 +++++++++++++++-- .../no_icmp_with_filters_and_vlan_accept.conf | 185 +++++++++++++++--- .../no_icmp_with_filters_and_vlan_drop.conf | 185 +++++++++++++++--- .../tests/data/no_icmp_with_filters_drop.conf | 181 +++++++++++++++-- .../tests/data/small_ipv4_mtu_icmp.conf | 173 ++++++++++++++-- .../tests/data/small_ipv6_mtu_no_icmp.conf | 173 ++++++++++++++-- .../data/small_ipv6_mtu_no_icmp_allow.conf | 173 ++++++++++++++-- .../small_ipv6_mtu_no_icmp_vlan_allow.conf | 177 +++++++++++++++-- .../lwaftr/tests/data/tunnel_icmp.conf | 173 ++++++++++++++-- .../tests/data/tunnel_icmp_without_mac4.conf | 173 ++++++++++++++-- .../tests/data/tunnel_icmp_withoutmac.conf | 173 ++++++++++++++-- src/program/lwaftr/tests/data/vlan.conf | 177 +++++++++++++++-- .../tests/data/vlan/big_mtu_no_icmp.conf | 177 +++++++++++++++-- .../lwaftr/tests/data/vlan/icmp_on_fail.conf | 177 +++++++++++++++-- .../lwaftr/tests/data/vlan/no_hairpin.conf | 177 +++++++++++++++-- .../lwaftr/tests/data/vlan/no_icmp.conf | 177 +++++++++++++++-- .../tests/data/vlan/no_icmp_maxfrags1.conf | 179 +++++++++++++++-- .../vlan/no_icmp_with_filters_accept.conf | 185 +++++++++++++++--- .../data/vlan/no_icmp_with_filters_drop.conf | 185 +++++++++++++++--- .../tests/data/vlan/small_ipv4_mtu_icmp.conf | 177 +++++++++++++++-- .../data/vlan/small_ipv6_mtu_no_icmp.conf | 177 +++++++++++++++-- .../vlan/small_ipv6_mtu_no_icmp_allow.conf | 177 +++++++++++++++-- .../lwaftr/tests/data/vlan/tunnel_icmp.conf | 177 +++++++++++++++-- .../data/vlan/tunnel_icmp_without_mac4.conf | 177 +++++++++++++++-- .../data/vlan/tunnel_icmp_withoutmac.conf | 177 +++++++++++++++-- src/program/lwaftr/tests/data/vlan/vlan.conf | 177 +++++++++++++++-- 34 files changed, 4900 insertions(+), 594 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 87e12ea11b..3d3cf5a098 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -29,7 +29,7 @@ end function run(args) local opts, conf_file, inv4_pcap, inv6_pcap = parse_args(args) - local conf = require('apps.lwaftr.conf').load_legacy_lwaftr_config(conf_file) + local conf = require('apps.lwaftr.conf').load_config(conf_file) local c = config.new() setup.load_bench(c, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') diff --git a/src/program/lwaftr/check/check.lua b/src/program/lwaftr/check/check.lua index 0c2440deea..0014ef2a9e 100644 --- a/src/program/lwaftr/check/check.lua +++ b/src/program/lwaftr/check/check.lua @@ -40,7 +40,8 @@ function run(args) or setup.load_check local conf_file, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap, counters_path = unpack(args) - local conf = lwconf.load_legacy_lwaftr_config(conf_file) + local conf = lwconf.load_lwaftr_config(conf_file) + for k,v in pairs(conf) do print(k, v) end local c = config.new() load_check(c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap) diff --git a/src/program/lwaftr/soaktest/soaktest.lua b/src/program/lwaftr/soaktest/soaktest.lua index c9fecce214..ad4106288d 100644 --- a/src/program/lwaftr/soaktest/soaktest.lua +++ b/src/program/lwaftr/soaktest/soaktest.lua @@ -39,7 +39,7 @@ function run (args) local load_soak_test = opts["on-a-stick"] and setup.load_soak_test_on_a_stick or setup.load_soak_test local c = config.new() - local conf = lwconf.load_legacy_lwaftr_config(conf_file) + local conf = lwconf.load_lwaftr_config(conf_file) load_soak_test(c, conf, inv4_pcap, inv6_pcap) engine.configure(c) diff --git a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf index ef4cc3fa4c..033ee1cc28 100644 --- a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1960, -ipv6_mtu = 2000, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1960; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 2000; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/icmp_on_fail.conf b/src/program/lwaftr/tests/data/icmp_on_fail.conf index 3ced37d2ae..403e4ac34b 100644 --- a/src/program/lwaftr/tests/data/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/icmp_on_fail.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/no_hairpin.conf b/src/program/lwaftr/tests/data/no_hairpin.conf index 747803702a..c0aa6ea8a3 100644 --- a/src/program/lwaftr/tests/data/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/no_hairpin.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = false, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/no_icmp.conf b/src/program/lwaftr/tests/data/no_icmp.conf index ea8273d1a6..89cbd401d9 100644 --- a/src/program/lwaftr/tests/data/no_icmp.conf +++ b/src/program/lwaftr/tests/data/no_icmp.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf index 3127520f56..a5cdf45ec2 100644 --- a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf @@ -1,19 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -max_fragments_per_reassembly_packet = 1, -max_ipv6_reassembly_packets = 10, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 1; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 1; + max-packets 10; + } +} diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf index dc5991d5d0..33eb8bd1de 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf @@ -1,21 +1,160 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -vlan_tagging = false, -ipv4_ingress_filter = "ip", -ipv4_egress_filter = "ip", -ipv6_ingress_filter = "ip6", -ipv6_egress_filter = "ip6" +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + egress-filter ip; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ingress-filter ip; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + egress-filter ip6; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ingress-filter ip6; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf index 8329d52073..7c918e85ec 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf @@ -1,23 +1,162 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true, -ipv4_ingress_filter = "ip", -ipv4_egress_filter = "ip", -ipv6_ingress_filter = "ip6", -ipv6_egress_filter = "ip6" +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + egress-filter ip; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ingress-filter ip; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + egress-filter ip6; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ingress-filter ip6; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf index dcd4c74f04..9607c978fa 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf @@ -1,23 +1,162 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true, -ipv4_ingress_filter = "len < 0", -ipv4_egress_filter = "len < 0", -ipv6_ingress_filter = "len < 0", -ipv6_egress_filter = "len < 0", +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ingress-filter "len < 0"; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf index 2b35520f6d..44c914d0f7 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf @@ -1,21 +1,160 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -vlan_tagging = false, -ipv4_ingress_filter = "len < 0", -ipv4_egress_filter = "len < 0", -ipv6_ingress_filter = "len < 0", -ipv6_egress_filter = "len < 0", +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ingress-filter "len < 0"; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf index ec2a1c1a9b..b563350552 100644 --- a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 576, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 576; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf index a34548e592..a4d9919346 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1500, -ipv6_mtu = 1280, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf index 85ee5f11aa..965536eb1a 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1500, -ipv6_mtu = 1280, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf index 71ffee6bf5..1776a5922f 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1500, -ipv6_mtu = 1280, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/tunnel_icmp.conf b/src/program/lwaftr/tests/data/tunnel_icmp.conf index 2b191643a6..d2d2033be1 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf index 47b59df918..0f3b0254eb 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop_ipv4_addr = 4.5.6.7 -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + ip 4.5.6.7; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf index a5f3cc025a..23f18efe35 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf @@ -1,17 +1,156 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop_ipv6_addr = 8:9:a:b:4:3:2:1 -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -vlan_tagging = false +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + ip 8:9:a:b:4:3:2:1; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/lwaftr/tests/data/vlan.conf b/src/program/lwaftr/tests/data/vlan.conf index 022f574f3b..5832b2ee52 100644 --- a/src/program/lwaftr/tests/data/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e3, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 6000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf index c1775cda23..9fe8b194a4 100644 --- a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1960, -ipv6_mtu = 2000, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1960; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 2000; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf index 59f96f6b6f..6ae8c9d674 100644 --- a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf index 250c98caa7..9dee35ff40 100644 --- a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = false, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp.conf b/src/program/lwaftr/tests/data/vlan/no_icmp.conf index 0e20f05f34..7a87407ce7 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf index 54f4d42bcf..a521ebc979 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf @@ -1,21 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -max_fragments_per_reassembly_packet = 1, -max_ipv6_reassembly_packets = 10, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 1; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 1; + max-packets 10; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf index d9e2f81c7e..7c918e85ec 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf @@ -1,23 +1,162 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -ipv4_ingress_filter = "ip", -ipv4_egress_filter = "ip", -ipv6_ingress_filter = "ip6", -ipv6_egress_filter = "ip6" -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + egress-filter ip; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ingress-filter ip; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + egress-filter ip6; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ingress-filter ip6; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf index 2d496542bc..9607c978fa 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf @@ -1,23 +1,162 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -ipv4_ingress_filter = "len < 0", -ipv4_egress_filter = "len < 0", -ipv6_ingress_filter = "len < 0", -ipv6_egress_filter = "len < 0", -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ingress-filter "len < 0"; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf index 84d41969b0..8b9c45aad6 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 576, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 576; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf index cdc2d5dba7..c8beaa2fe9 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1500, -ipv6_mtu = 1280, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = DROP, -policy_icmpv6_incoming = DROP, -policy_icmpv4_outgoing = DROP, -policy_icmpv6_outgoing = DROP, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf index 71ffee6bf5..1776a5922f 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1500, -ipv6_mtu = 1280, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf index cbf0e11b85..d708bd3048 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf index 8cef46cb1d..d39da3969f 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop_ipv4_addr = 4.5.6.7 -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -vlan_tagging = true, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + ip 4.5.6.7; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf index 2550443ee4..790ebc797a 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e5, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop_ipv6_addr = 8:9:a:b:4:3:2:1 -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + ip 8:9:a:b:4:3:2:1; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} diff --git a/src/program/lwaftr/tests/data/vlan/vlan.conf b/src/program/lwaftr/tests/data/vlan/vlan.conf index 022f574f3b..5832b2ee52 100644 --- a/src/program/lwaftr/tests/data/vlan/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan/vlan.conf @@ -1,19 +1,158 @@ -aftr_ipv4_ip = 10.10.10.10, -aftr_ipv6_ip = 8:9:a:b:c:d:e:f, -aftr_mac_b4_side = 22:22:22:22:22:22, -aftr_mac_inet_side = 12:12:12:12:12:12, -binding_table = binding-table.txt, -hairpinning = true, -icmpv6_rate_limiter_n_packets=6e3, -icmpv6_rate_limiter_n_seconds=2, -inet_mac = 68:68:68:68:68:68, -ipv4_mtu = 1460, -ipv6_mtu = 1500, -next_hop6_mac = 44:44:44:44:44:44, -policy_icmpv4_incoming = ALLOW, -policy_icmpv6_incoming = ALLOW, -policy_icmpv4_outgoing = ALLOW, -policy_icmpv6_outgoing = ALLOW, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + reserved-ports-bit-count 0; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + reserved-ports-bit-count 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + reserved-ports-bit-count 0; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 178.79.150.3; + padding 0; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.1; + padding 0; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + br 0; + } + softwire { + ipv4 178.79.150.15; + padding 0; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + br 0; + } + softwire { + ipv4 178.79.150.233; + padding 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + br 0; + } + softwire { + ipv4 178.79.150.2; + padding 0; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 6000; + period 2; + } + generate-icmp-errors true; + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} From 3f7f744c95d774531510cba93c8b3ec4bb0f310a Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Wed, 9 Nov 2016 12:51:47 +0100 Subject: [PATCH 167/631] Add an example of running a load generator towards on-a-stick mode (#554) --- src/program/lwaftr/doc/README.running.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/program/lwaftr/doc/README.running.md b/src/program/lwaftr/doc/README.running.md index 9224295ac9..9691a3ea0e 100644 --- a/src/program/lwaftr/doc/README.running.md +++ b/src/program/lwaftr/doc/README.running.md @@ -74,6 +74,18 @@ $ sudo ./snabb lwaftr run \ --on-a-stick 0000:02:00.1 ``` +You can run a load generator: + +``` +$ sudo ./snabb lwaftr loadtest \ + program/lwaftr/tests/benchdata/ipv4-0550.pcap IPv4 IPv6 0000:82:00.0 \ + program/lwaftr/tests/benchdata/ipv6-0550.pcap IPv6 IPv4 0000:82:00.1 +``` + +For the main lwaftr instance to receive any traffic, either 82:00.0 or 82:00.1 +should be wired to 02:00.1 . Since `lwaftr loadtest` does not currently have +an on-a-stick mode, the main lwaftr instance will only see half the loadtest +traffic. ## The packetblaster From fc4c711741c64349f3dee81c8f8daeeb61df6f02 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 13:46:25 +0100 Subject: [PATCH 168/631] Migrate snabbvmx lwaftr to new configuration --- src/program/snabbvmx/lwaftr/lwaftr.lua | 2 +- src/program/snabbvmx/lwaftr/setup.lua | 2 +- .../tests/conf/snabbvmx-lwaftr-xe0.conf | 1386 +++++++++++++++- .../snabbvmx/tests/conf/snabbvmx-lwaftr.conf | 1387 ++++++++++++++++- .../end-to-end/data/snabbvmx-lwaftr-xe1.conf | 92 +- .../data/vlan/snabbvmx-lwaftr-xe1.conf | 97 +- 6 files changed, 2906 insertions(+), 60 deletions(-) diff --git a/src/program/snabbvmx/lwaftr/lwaftr.lua b/src/program/snabbvmx/lwaftr/lwaftr.lua index 24f7094f1f..d7c9e9559e 100644 --- a/src/program/snabbvmx/lwaftr/lwaftr.lua +++ b/src/program/snabbvmx/lwaftr/lwaftr.lua @@ -106,7 +106,7 @@ function run(args) if not file_exists(conf.lwaftr) then fatal(("lwAFTR conf file '%s' not found"):format(conf.lwaftr)) end - lwconf = require('apps.lwaftr.conf').load_legacy_lwaftr_config(conf.lwaftr) + lwconf = require('apps.lwaftr.conf').load_lwaftr_config(conf.lwaftr) -- If one interface has vlan tags, the other one should as well. assert((not lwconf.external_interface.vlan_tag) == (not lwconf.internal_interface.vlan_tag)) diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index 818f244073..4947cdf43e 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -317,7 +317,7 @@ local function load_conf (conf_filename) if not file_exists(filename) then filename = lib.dirname(conf_filename).."/"..filename end - return require("apps.lwaftr.conf").load_legacy_lwaftr_config(filename) + return require("apps.lwaftr.conf").load_lwaftr_config(filename) end local conf = dofile(conf_filename) return conf, load_lwaftr_config(conf, conf_filename) diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf index 9712612c79..2fae04e4de 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf @@ -1,16 +1,1370 @@ -binding_table = binding_table.txt, -vlan_tagging = false, -aftr_ipv6_ip = fc00::100, -aftr_mac_inet_side = 02:AA:AA:AA:AA:AA, -inet_mac = 02:99:99:99:99:99, -ipv6_mtu = 9500, -policy_icmpv6_incoming = DROP, -policy_icmpv6_outgoing = DROP, -icmpv6_rate_limiter_n_packets = 6e5, -icmpv6_rate_limiter_n_seconds = 2, -aftr_ipv4_ip = 10.0.1.1, -aftr_mac_b4_side = 02:AA:AA:AA:AA:AA, -next_hop6_mac = 02:99:99:99:99:99, -ipv4_mtu = 1460, -policy_icmpv4_incoming = DROP, -policy_icmpv4_outgoing = DROP, +binding-table { + br-address fc00::100; + psid-map { + addr 193.5.1.100; + end-addr 193.5.1.102; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ad; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:b1; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:c4; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:132; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:12c; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:109; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:fb; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:11d; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:111; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:a4; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:c3; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:100; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:137; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:e4; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:bf; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:d5; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:133; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:12d; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:9e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:f9; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:135; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:ef; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:99; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:a9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:d7; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:a8; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:91; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:da; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:de; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:a6; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:a0; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:be; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:105; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:10c; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:b4; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:d3; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:c9; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:81; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:fc; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:101; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:11b; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:86; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:d2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:123; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:122; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:12a; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:96; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:e9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:e3; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:12e; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:126; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:89; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ed; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:106; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:d0; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:10e; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:11f; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:103; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:121; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:7f; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:104; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:95; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:bc; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:f5; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:9a; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:97; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:120; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:117; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:88; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:ba; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:7e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:c1; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:108; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:125; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ec; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:94; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:9b; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ab; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:9f; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:93; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:13a; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:c5; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:cc; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:df; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:f1; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:116; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:131; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:f8; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:10a; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:d6; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:aa; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:db; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:8a; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:c0; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:87; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:136; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:a7; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:c2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:134; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:cf; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:8e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:fa; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:9d; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:e5; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:fe; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:b9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:f3; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:af; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:f4; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:12f; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:b7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:11c; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:d8; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:127; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ae; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:107; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:8b; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:92; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:e6; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:d4; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:82; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:f2; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:8c; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:a5; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:f7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:129; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:10d; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:110; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:a1; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:ce; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:112; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:b2; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:ac; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:c7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:10f; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:bd; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:90; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:12b; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:f0; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:113; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:b0; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:dc; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:11e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:ee; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:11a; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:a2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:fd; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:f6; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:e2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:124; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:83; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:ca; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:b6; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:118; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:8d; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:139; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:115; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:b5; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:dd; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:84; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:d1; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:8f; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:bb; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:98; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:80; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:c8; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:85; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:119; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:ff; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:b3; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:10b; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:e1; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:9c; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:e7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:114; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:cb; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:102; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:128; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:a3; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ea; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:138; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:e0; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:d9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:eb; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:e8; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:130; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:cd; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:b8; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:c6; + br 0; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.0.1.1; + mac 02:aa:aa:aa:aa:aa; + mtu 1460; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip fc00::100; + mac 02:aa:aa:aa:aa:aa; + mtu 9500; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf index 5047a8d81f..2fae04e4de 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf @@ -1,17 +1,1370 @@ -binding_table = binding_table.txt, -vlan_tagging = false, -aftr_ipv6_ip = fc00::100, -aftr_mac_inet_side = 02:AA:AA:AA:AA:AA, -inet_mac = 02:99:99:99:99:99, -ipv6_mtu = 9500, -policy_icmpv6_incoming = DROP, -policy_icmpv6_outgoing = DROP, -icmpv6_rate_limiter_n_packets = 6e5, -icmpv6_rate_limiter_n_seconds = 2, -aftr_ipv4_ip = 10.0.1.1, -aftr_mac_b4_side = 02:AA:AA:AA:AA:AA, -# b4_mac = 02:99:99:99:99:99, -next_hop6_mac = 02:99:99:99:99:99, -ipv4_mtu = 1460, -policy_icmpv4_incoming = DROP, -policy_icmpv4_outgoing = DROP, +binding-table { + br-address fc00::100; + psid-map { + addr 193.5.1.100; + end-addr 193.5.1.102; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ad; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:b1; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:c4; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:132; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:12c; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:109; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:fb; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:11d; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:111; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:a4; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:c3; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:100; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:137; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:e4; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:bf; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:d5; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:133; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:12d; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:9e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:f9; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:135; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:ef; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:99; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:a9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:d7; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:a8; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:91; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:da; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:de; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:a6; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:a0; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:be; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:105; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:10c; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:b4; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:d3; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:c9; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:81; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:fc; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:101; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:11b; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:86; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:d2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:123; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:122; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:12a; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:96; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:e9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:e3; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:12e; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:126; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:89; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ed; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:106; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:d0; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:10e; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:11f; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:103; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:121; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:7f; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:104; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:95; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:bc; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:f5; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:9a; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:97; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:120; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:117; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:88; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:ba; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:7e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:c1; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:108; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:125; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ec; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:94; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:9b; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ab; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:9f; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:93; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:13a; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:c5; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:cc; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:df; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:f1; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:116; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:131; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:f8; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:10a; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:d6; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:aa; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:db; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:8a; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:c0; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:87; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:136; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:a7; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:c2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:134; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:cf; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:8e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:fa; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:9d; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:e5; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:fe; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:b9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:f3; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:af; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:f4; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:12f; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:b7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:11c; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:d8; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:127; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ae; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:107; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:8b; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:92; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:e6; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:d4; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:82; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:f2; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:8c; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:a5; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:f7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:129; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:10d; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:110; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:a1; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:ce; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:112; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:b2; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:ac; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:c7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:10f; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:bd; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:90; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:12b; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:f0; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:113; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:b0; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:dc; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:11e; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:ee; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:11a; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:a2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:fd; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:f6; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:e2; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:124; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:83; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:ca; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:b6; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:118; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:8d; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:139; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:115; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:b5; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:dd; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:84; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:d1; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:8f; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:bb; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:98; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:80; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:c8; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:85; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:119; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:ff; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:b3; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:10b; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:e1; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:9c; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:e7; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:114; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:cb; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:102; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:128; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:a3; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ea; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:138; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:e0; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:d9; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:eb; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:e8; + br 0; + } + softwire { + ipv4 193.5.1.102; + padding 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:130; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:cd; + br 0; + } + softwire { + ipv4 193.5.1.100; + padding 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:b8; + br 0; + } + softwire { + ipv4 193.5.1.101; + padding 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:c6; + br 0; + } +} +external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + ip 10.0.1.1; + mac 02:aa:aa:aa:aa:aa; + mtu 1460; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors false; + hairpinning true; + ip fc00::100; + mac 02:aa:aa:aa:aa:aa; + mtu 9500; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf index 0d6de84a69..365c379f79 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf @@ -1,11 +1,81 @@ -hairpinning = false, -vlan_tagging = false, -binding_table = binding_table.txt.s, -aftr_ipv6_ip = fc00:168:10::2, -aftr_ipv4_ip = 192.168.10.2, -aftr_mac_inet_side = 02:cf:69:15:81:01, -inet_mac = 90:e2:ba:94:2a:bc -aftr_mac_b4_side = 02:cf:69:15:81:01, -next_hop6_mac = 90:e2:ba:94:2a:bc, -ipv4_mtu = 9000, -ipv6_mtu = 9000, +binding-table { + br-address 2a02:587:f700::100; + psid-map { + addr 10.10.0.10; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + psid-map { + addr 10.10.0.0; + end-addr 10.10.0.1; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 1; + b4-ipv6 2a02:587:f710::400; + br 0; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 4; + b4-ipv6 2a02:587:f710::430; + br 0; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 3; + b4-ipv6 2a02:587:f710::420; + br 0; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 2; + b4-ipv6 2a02:587:f710::410; + br 0; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 192.168.10.2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning false; + ip fc00:168:10::2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} diff --git a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf index 88d6d1f85d..1f70ca971b 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf @@ -1,14 +1,83 @@ -hairpinning = false, -vlan_tagging = false, -binding_table = binding_table.txt.s, -aftr_ipv6_ip = fc00:168:10::2, -aftr_ipv4_ip = 192.168.10.2, -aftr_mac_inet_side = 02:cf:69:15:81:01, -inet_mac = 90:e2:ba:94:2a:bc -aftr_mac_b4_side = 02:cf:69:15:81:01, -next_hop6_mac = 90:e2:ba:94:2a:bc, -ipv4_mtu = 9000, -ipv6_mtu = 9000, -v4_vlan_tag = 1092, # 0x444 -v6_vlan_tag = 1638, # 0x666 -vlan_tagging = true +binding-table { + br-address 2a02:587:f700::100; + psid-map { + addr 10.10.0.10; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + psid-map { + addr 10.10.0.0; + end-addr 10.10.0.1; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 1; + b4-ipv6 2a02:587:f710::400; + br 0; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 4; + b4-ipv6 2a02:587:f710::430; + br 0; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 3; + b4-ipv6 2a02:587:f710::420; + br 0; + } + softwire { + ipv4 10.10.0.0; + padding 0; + psid 2; + b4-ipv6 2a02:587:f710::410; + br 0; + } +} +external-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + ip 192.168.10.2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1092; +} +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + hairpinning false; + ip fc00:168:10::2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } + vlan-tag 1638; +} From 8019d7d9e88690e11440b58ea0c6fd624e2a7042 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 14:21:25 +0100 Subject: [PATCH 169/631] Fix ctable lookup_and_copy bug --- src/lib/ctable.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 8a3ed8fec0..095f450d26 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -298,7 +298,7 @@ end function CTable:lookup_and_copy(key, entry) local entry_ptr = self:lookup_ptr(key) - if not ptr then return false end + if not entry_ptr then return false end entry = entry_ptr return true end From 31beaeb6ee0e3362a5c8529fc7cd30ec5d2912b4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 14:21:45 +0100 Subject: [PATCH 170/631] Add hash and ctable subcommands to snabbmark --- src/program/snabbmark/README | 6 ++ src/program/snabbmark/snabbmark.lua | 131 ++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/src/program/snabbmark/README b/src/program/snabbmark/README index 8a21bb8940..aa4984fb21 100644 --- a/src/program/snabbmark/README +++ b/src/program/snabbmark/README @@ -43,3 +43,9 @@ Usage: latter being the default. Optionally, a LuaJIT profiler option string can be supplied as , which will cause the benchmark run to be profiled accordingly. + + snabbmark hash + Benchmark hash functions used for internal data structures. + + snabbmark ctable + Benchmark insertion and lookup for the "ctable" data structure. diff --git a/src/program/snabbmark/snabbmark.lua b/src/program/snabbmark/snabbmark.lua index aff47256e1..4d9814a76c 100644 --- a/src/program/snabbmark/snabbmark.lua +++ b/src/program/snabbmark/snabbmark.lua @@ -23,6 +23,10 @@ function run (args) intel1g(unpack(args)) elseif command == 'esp' and #args >= 2 then esp(unpack(args)) + elseif command == 'hash' and #args == 0 then + hash(unpack(args)) + elseif command == 'ctable' and #args == 0 then + ctable(unpack(args)) else print(usage) main.exit(1) @@ -390,3 +394,130 @@ function esp (npackets, packet_size, mode, profile) :format(packet_size, gbits(bps))) end end + +local pmu = require('lib.pmu') +local has_pmu_counters, err = pmu.is_available() +if not has_pmu_counters then + io.stderr:write('No PMU available: '..err..'\n') +end + +if has_pmu_counters then pmu.setup() end + +local function measure(f, iterations) + local set + if has_pmu_counters then set = pmu.new_counter_set() end + local start = C.get_time_ns() + if has_pmu_counters then pmu.switch_to(set) end + local res = f(iterations) + if has_pmu_counters then pmu.switch_to(nil) end + local stop = C.get_time_ns() + local ns = tonumber(stop-start) + local cycles = nil + if has_pmu_counters then cycles = pmu.to_table(set).cycles end + return cycles, ns, res +end + +local function test_perf(f, iterations, what) + require('jit').flush() + io.write(tostring(what or f)..': ') + io.flush() + local cycles, ns, res = measure(f, iterations) + if cycles then + cycles = cycles/iterations + io.write(('%.2f cycles, '):format(cycles)) + end + ns = ns/iterations + io.write(('%.2f ns per iteration (result: %s)\n'):format( + ns, tostring(res))) + return res +end + +function hash () + local murmur = require('lib.hash.murmur').MurmurHash3_x86_32:new() + local vptr = ffi.new("uint8_t [4]") + local uint32_ptr_t = ffi.typeof('uint32_t*') + local lshift = require('bit').lshift + local INT32_MIN = -0x80000000 + function murmur_hash_32(u32) + ffi.cast(uint32_ptr_t, vptr)[0] = u32 + return murmur:hash(vptr, 4, 0ULL).u32[0] + end + + local jenkins_hash_32 = require('lib.ctable').hash_32 + local function test_jenkins(iterations) + local result + for i=1,iterations do result=jenkins_hash_32(i) end + return result + end + + local function test_murmur(iterations) + local result + for i=1,iterations do result=murmur_hash_32(i) end + return result + end + + test_perf(test_jenkins, 1e8, 'jenkins hash') + test_perf(test_murmur, 1e8, 'murmur hash (32 bit)') +end + +function ctable () + local ctable = require('lib.ctable') + local bnot = require('bit').bnot + local ctab = ctable.new( + { key_type = ffi.typeof('uint32_t'), + value_type = ffi.typeof('int32_t[6]'), + hash_fn = ctable.hash_32 }) + local occupancy = 2e6 + ctab:resize(occupancy / 0.4 + 1) + + local function test_insertion(count) + local v = ffi.new('int32_t[6]'); + for i = 1,count do + for j=0,5 do v[j] = bnot(i) end + ctab:add(i, v) + end + end + + local function test_lookup_ptr(count) + local result = ctab.entry_type() + for i = 1, count do + result = ctab:lookup_ptr(i) + end + return result + end + + local function test_lookup_and_copy(count) + local result = ctab.entry_type() + for i = 1, count do + ctab:lookup_and_copy(i, result) + end + return result + end + + test_perf(test_insertion, occupancy, 'insertion (40% occupancy)') + test_perf(test_lookup_ptr, occupancy, 'lookup_ptr (40% occupancy)') + test_perf(test_lookup_and_copy, occupancy, 'lookup_and_copy (40% occupancy)') + + local stride = 1 + repeat + local streamer = ctab:make_lookup_streamer(stride) + local function test_lookup_streamer(count) + local result + for i = 1, count, stride do + local n = math.min(stride, count-i+1) + for j = 0, n-1 do + streamer.entries[j].key = i + j + end + streamer:stream() + result = streamer.entries[n-1].value[0] + end + return result + end + -- Note that "result" is part of the value, not an index into + -- the table, and so we expect the results to be different from + -- ctab:lookup(). + test_perf(test_lookup_streamer, occupancy, + 'streaming lookup, stride='..stride) + stride = stride * 2 + until stride > 256 +end From bf4471a80280d353784b66ef2e2b073ec467e449 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 14:23:26 +0100 Subject: [PATCH 171/631] Remove PodHashMap So long and thanks for all the fish! --- src/apps/lwaftr/podhashmap.lua | 651 ------------------ src/apps/lwaftr/test_phm_create.lua | 67 -- src/apps/lwaftr/test_phm_lookup.lua | 40 -- src/apps/lwaftr/test_phm_streaming_lookup.lua | 60 -- 4 files changed, 818 deletions(-) delete mode 100644 src/apps/lwaftr/podhashmap.lua delete mode 100644 src/apps/lwaftr/test_phm_create.lua delete mode 100644 src/apps/lwaftr/test_phm_lookup.lua delete mode 100644 src/apps/lwaftr/test_phm_streaming_lookup.lua diff --git a/src/apps/lwaftr/podhashmap.lua b/src/apps/lwaftr/podhashmap.lua deleted file mode 100644 index 89a2850e5e..0000000000 --- a/src/apps/lwaftr/podhashmap.lua +++ /dev/null @@ -1,651 +0,0 @@ -module(..., package.seeall) - -local ffi = require("ffi") -local C = ffi.C -local S = require("syscall") -local bit = require("bit") -local bxor, bnot = bit.bxor, bit.bnot -local tobit, lshift, rshift = bit.tobit, bit.lshift, bit.rshift -local max, floor, ceil = math.max, math.floor, math.ceil - -PodHashMap = {} -LookupStreamer = {} - -local HASH_MAX = 0xFFFFFFFF -local INT32_MIN = -0x80000000 -local INITIAL_SIZE = 8 -local MAX_OCCUPANCY_RATE = 0.9 -local MIN_OCCUPANCY_RATE = 0.0 - -local function make_entry_type(key_type, value_type) - return ffi.typeof([[struct { - uint32_t hash; - $ key; - $ value; - } __attribute__((packed))]], - key_type, - value_type) -end - --- hash := [0,HASH_MAX); scale := size/HASH_MAX -local function hash_to_index(hash, scale) - return floor(hash*scale + 0.5) -end - -local function make_equal_fn(key_type) - local size = ffi.sizeof(key_type) - local cast = ffi.cast - if tonumber(ffi.new(key_type)) then - return function (a, b) - return a == b - end - elseif size == 4 then - local uint32_ptr_t = ffi.typeof('uint32_t*') - return function (a, b) - return cast(uint32_ptr_t, a)[0] == cast(uint32_ptr_t, b)[0] - end - elseif size == 8 then - local uint64_ptr_t = ffi.typeof('uint64_t*') - return function (a, b) - return cast(uint64_ptr_t, a)[0] == cast(uint64_ptr_t, b)[0] - end - else - return function (a, b) - return C.memcmp(a, b, size) == 0 - end - end -end - -function PodHashMap.new(key_type, value_type, hash_fn) - local phm = {} - phm.entry_type = make_entry_type(key_type, value_type) - phm.hash_fn = hash_fn - phm.equal_fn = make_equal_fn(key_type) - phm.size = 0 - phm.occupancy = 0 - phm.max_occupancy_rate = MAX_OCCUPANCY_RATE - phm.min_occupancy_rate = MIN_OCCUPANCY_RATE - phm = setmetatable(phm, { __index = PodHashMap }) - phm:resize(INITIAL_SIZE) - return phm -end - --- FIXME: There should be a library to help allocate anonymous --- hugepages, not this code. -local try_huge_pages = true -local huge_page_threshold = 1e6 -local function calloc(t, count) - local byte_size = ffi.sizeof(t) * count - local mem, err - if try_huge_pages and byte_size > huge_page_threshold then - mem, err = S.mmap(nil, byte_size, 'read, write', - 'private, anonymous, hugetlb') - if not mem then - print("hugetlb mmap failed ("..tostring(err)..'), falling back.') - -- FIXME: Increase vm.nr_hugepages. See - -- core.memory.reserve_new_page(). - end - end - if not mem then - mem, err = S.mmap(nil, byte_size, 'read, write', - 'private, anonymous') - if not mem then error("mmap failed: " .. tostring(err)) end - end - local ret = ffi.cast(ffi.typeof('$*', t), mem) - ffi.gc(ret, function (ptr) S.munmap(ptr, byte_size) end) - return ret -end - -local header_t = ffi.typeof[[ -struct { - uint32_t size; - uint32_t entry_size; - uint32_t occupancy; - uint32_t max_displacement; - double max_occupancy_rate; - double min_occupancy_rate; -} -]] - -function PodHashMap:save(stream) - stream:write_ptr(header_t(self.size, ffi.sizeof(self.entry_type), - self.occupancy, self.max_displacement, - self.max_occupancy_rate, self.min_occupancy_rate)) - stream:write_array(self.entries, - self.entry_type, - self.size + self.max_displacement + 1) -end - -function load(stream, key_t, value_t, hash_fn) - local map = PodHashMap.new(key_t, value_t, hash_fn) - local header = stream:read_ptr(header_t) - assert(header.entry_size == header.entry_size) - - map.size = header.size - map.scale = map.size / HASH_MAX - map.occupancy = header.occupancy - map.max_displacement = header.max_displacement - map.min_occupancy_rate = header.min_occupancy_rate - map.max_occupancy_rate = header.max_occupancy_rate - map.occupancy_hi = ceil(map.size * map.max_occupancy_rate) - map.occupancy_lo = floor(map.size * map.min_occupancy_rate) - local entry_count = map.size + map.max_displacement + 1 - map.entries = calloc(map.entry_type, entry_count) - - -- Copy the entries from disk into hugepages. - C.memcpy(map.entries, - stream:read_array(map.entry_type, entry_count), - ffi.sizeof(map.entry_type) * entry_count) - - return map -end - -function PodHashMap:resize(size) - assert(size >= (self.occupancy / self.max_occupancy_rate)) - local old_entries = self.entries - local old_size = self.size - - self.size = size - self.scale = self.size / HASH_MAX - self.occupancy = 0 - self.max_displacement = 0 - self.occupancy_hi = ceil(self.size * self.max_occupancy_rate) - self.occupancy_lo = floor(self.size * self.min_occupancy_rate) - -- Allocate double the requested number of entries to make sure there - -- is sufficient displacement if all hashes map to the last bucket. - self.entries = calloc(self.entry_type, size * 2) - - for i=0,self.size*2-1 do self.entries[i].hash = HASH_MAX end - - for i=0,old_size*2-1 do - if old_entries[i].hash ~= HASH_MAX then - self:insert(old_entries[i].hash, old_entries[i].key, old_entries[i].value) - end - end -end - -function PodHashMap:insert(hash, key, value) - if self.occupancy + 1 > self.occupancy_hi then - self:resize(self.size * 2) - end - - local entries = self.entries - local scale = self.scale - local start_index = hash_to_index(hash, self.scale) - local index = start_index - --print('adding ', hash, key, value, index) - - while entries[index].hash < hash do - --print('displace', index, entries[index].hash) - index = index + 1 - end - - while entries[index].hash == hash do - --- Update currently unsupported. - --print('update?', index) - assert(not self.equal_fn(key, entries[index].key)) - index = index + 1 - end - - self.max_displacement = max(self.max_displacement, index - start_index) - - if entries[index].hash ~= HASH_MAX then - --- Rob from rich! - --print('steal', index) - local empty = index; - while entries[empty].hash ~= HASH_MAX do empty = empty + 1 end - --print('end', empty) - while empty > index do - entries[empty] = entries[empty - 1] - local displacement = empty - hash_to_index(entries[empty].hash, scale) - self.max_displacement = max(self.max_displacement, displacement) - empty = empty - 1; - end - end - - self.occupancy = self.occupancy + 1 - entries[index].hash = hash - entries[index].key = key - entries[index].value = value - return index -end - -function PodHashMap:add(key, value) - local hash = self.hash_fn(key) - assert(hash ~= HASH_MAX) - return self:insert(hash, key, value) -end - -function PodHashMap:lookup(key) - local hash = self.hash_fn(key) - local entries = self.entries - local index = hash_to_index(hash, self.scale) - local other_hash = entries[index].hash - - if hash == other_hash and self.equal_fn(key, entries[index].key) then - -- Found! - return index - end - - while other_hash < hash do - index = index + 1 - other_hash = entries[index].hash - end - - while other_hash == hash do - if self.equal_fn(key, entries[index].key) then - -- Found! - return index - end - -- Otherwise possibly a collision. - index = index + 1 - other_hash = entries[index].hash - end - - -- Not found. - return nil -end - --- FIXME: Does NOT shrink max_displacement -function PodHashMap:remove_at(i) - assert(not self:is_empty(i)) - - local entries = self.entries - local scale = self.scale - - self.occupancy = self.occupancy - 1 - entries[i].hash = HASH_MAX - - while true do - local next = i + 1 - local next_hash = entries[next].hash - if next_hash == HASH_MAX then break end - if hash_to_index(next_hash, scale) == next then break end - -- Give to the poor. - entries[i] = entries[next] - entries[next].hash = HASH_MAX - i = next - end - - if self.occupancy < self.size * self.occupancy_lo then - self:resize(self.size / 2) - end -end - -function PodHashMap:is_empty(i) - assert(i >= 0 and i < self.size*2) - return self.entries[i].hash == HASH_MAX -end - -function PodHashMap:hash_at(i) - assert(not self:is_empty(i)) - return self.entries[i].hash -end - -function PodHashMap:key_at(i) - assert(not self:is_empty(i)) - return self.entries[i].key -end - -function PodHashMap:val_at(i) - assert(not self:is_empty(i)) - return self.entries[i].value -end - -function PodHashMap:selfcheck() - local occupancy = 0 - local max_displacement = 0 - - local function fail(expected, op, found, what, where) - if where then where = 'at '..where..': ' else where = '' end - error(where..what..' check: expected '..expected..op..'found '..found) - end - local function expect_eq(expected, found, what, where) - if expected ~= found then fail(expected, '==', found, what, where) end - end - local function expect_le(expected, found, what, where) - if expected > found then fail(expected, '<=', found, what, where) end - end - - local prev = 0 - for i = 0,self.size*2-1 do - local entry = self.entries[i] - local hash = entry.hash - if hash ~= 0xffffffff then - expect_eq(self.hash_fn(entry.key), hash, 'hash', i) - local index = hash_to_index(hash, self.scale) - if prev == 0xffffffff then - expect_eq(index, i, 'undisplaced index', i) - else - expect_le(prev, hash, 'displaced hash', i) - end - occupancy = occupancy + 1 - max_displacement = max(max_displacement, i - index) - end - prev = hash - end - - expect_eq(occupancy, self.occupancy, 'occupancy') - -- Compare using <= because remove_at doesn't update max_displacement. - expect_le(max_displacement, self.max_displacement, 'max_displacement') -end - -function PodHashMap:dump() - local function dump_one(index) - io.write(index..':') - local entry = self.entries[index] - if (entry.hash == HASH_MAX) then - io.write('\n') - else - local distance = index - hash_to_index(entry.hash, self.scale) - io.write(' hash: '..entry.hash..' (distance: '..distance..')\n') - io.write(' key: '..tostring(entry.key)..'\n') - io.write(' value: '..tostring(entry.value)..'\n') - end - end - for index=0,self.size-1 do dump_one(index) end - for index=self.size,self.size*2-1 do - if self.entries[index].hash == HASH_MAX then break end - dump_one(index) - end -end - -function PodHashMap:iterate() - local max_entry = self.entries + self.size + self.max_displacement - local function next_entry(max_entry, entry) - while entry < max_entry do - entry = entry + 1 - if entry.hash ~= HASH_MAX then return entry end - end - end - return next_entry, max_entry, self.entries - 1 -end - -function PodHashMap:make_lookup_streamer(stride) - -- These requires are here because they rely on dynasm, which the - -- user might not have. In that case, since they get no benefit from - -- streaming lookup, restrict them to the scalar lookup. - local binary_search = require('lib.binary_search') - local multi_copy = require('lib.multi_copy') - - local res = { - all_entries = self.entries, - stride = stride, - hash_fn = self.hash_fn, - equal_fn = self.equal_fn, - entries_per_lookup = self.max_displacement + 1, - scale = self.scale, - pointers = ffi.new('void*['..stride..']'), - entries = calloc(self.entry_type, stride), - -- Binary search over N elements can return N if no entry was - -- found that was greater than or equal to the key. We would - -- have to check the result of binary search to ensure that we - -- are reading a value in bounds. To avoid this, allocate one - -- more entry. - stream_entries = calloc(self.entry_type, - stride * (self.max_displacement + 1) + 1) - } - -- Give res.pointers sensible default values in case the first lookup - -- doesn't fill the pointers vector. - for i = 0, stride-1 do res.pointers[i] = self.entries end - - -- Initialize the stream_entries to HASH_MAX for sanity. - for i = 0, stride * (self.max_displacement + 1) do - res.stream_entries[i].hash = HASH_MAX - end - - -- Compile multi-copy and binary-search procedures that are - -- specialized for this table and this stride. - local entry_size = ffi.sizeof(self.entry_type) - res.multi_copy = multi_copy.gen(stride, res.entries_per_lookup * entry_size) - res.binary_search = binary_search.gen(res.entries_per_lookup, self.entry_type) - - return setmetatable(res, { __index = LookupStreamer }) -end - -function LookupStreamer:stream() - local stride = self.stride - local entries = self.entries - local pointers = self.pointers - local stream_entries = self.stream_entries - local entries_per_lookup = self.entries_per_lookup - local equal_fn = self.equal_fn - - for i=0,stride-1 do - local hash = self.hash_fn(entries[i].key) - local index = hash_to_index(hash, self.scale) - entries[i].hash = hash - pointers[i] = self.all_entries + index - end - - self.multi_copy(stream_entries, pointers) - - -- Copy results into entries. - for i=0,stride-1 do - local hash = entries[i].hash - local index = i * entries_per_lookup - local found = self.binary_search(stream_entries + index, hash) - -- It could be that we read one beyond the ENTRIES_PER_LOOKUP - -- entries allocated for this key; that's fine. See note in - -- make_lookup_streamer. - if found.hash == hash then - -- Direct hit? - if equal_fn(found.key, entries[i].key) then - entries[i].value = found.value - else - -- Mark this result as not found unless we prove - -- otherwise. - entries[i].hash = HASH_MAX - - -- Collision? - found = found + 1 - while found.hash == hash do - if equal_fn(found.key, entries[i].key) then - -- Yay! Re-mark this result as found. - entries[i].hash = hash - entries[i].value = found.value - break - end - found = found + 1 - end - end - else - -- Not found. - entries[i].hash = HASH_MAX - end - end -end - -function LookupStreamer:is_empty(i) - assert(i >= 0 and i < self.stride) - return self.entries[i].hash == HASH_MAX -end - -function LookupStreamer:is_found(i) - return not self:is_empty(i) -end - --- One of Bob Jenkins' hashes from --- http://burtleburtle.net/bob/hash/integer.html. Chosen to result --- in the least badness as we adapt to int32 bitops. -function hash_i32(i32) - i32 = tobit(i32) - i32 = i32 + bnot(lshift(i32, 15)) - i32 = bxor(i32, (rshift(i32, 10))) - i32 = i32 + lshift(i32, 3) - i32 = bxor(i32, rshift(i32, 6)) - i32 = i32 + bnot(lshift(i32, 11)) - i32 = bxor(i32, rshift(i32, 16)) - - -- Unset the low bit, to distinguish valid hashes from HASH_MAX. - i32 = lshift(i32, 1) - -- Project result to u32 range. - return i32 - INT32_MIN -end - -local murmur = require('lib.hash.murmur').MurmurHash3_x86_32:new() -local vptr = ffi.new("uint8_t [4]") -function murmur_hash_i32(i32) - ffi.cast("int32_t*", vptr)[0] = i32 - local h = murmur:hash(vptr, 4, 0ULL).u32[0] - - -- Unset the low bit, to distinguish valid hashes from HASH_MAX. - local i32 = lshift(i32, 1) - -- Project result to u32 range. - return i32 - INT32_MIN -end - -function selftest() - local pmu = require('lib.pmu') - local has_pmu_counters, err = pmu.is_available() - if not has_pmu_counters then - print('No PMU available: '..err) - end - - if has_pmu_counters then pmu.setup() end - - local function measure(f, iterations) - local set - if has_pmu_counters then set = pmu.new_counter_set() end - local start = C.get_time_ns() - if has_pmu_counters then pmu.switch_to(set) end - local res = f(iterations) - if has_pmu_counters then pmu.switch_to(nil) end - local stop = C.get_time_ns() - local ns = tonumber(stop-start) - local cycles = nil - if has_pmu_counters then cycles = pmu.to_table(set).cycles end - return cycles, ns, res - end - - local function check_perf(f, iterations, max_cycles, max_ns, what) - require('jit').flush() - io.write(tostring(what or f)..': ') - io.flush() - local cycles, ns, res = measure(f, iterations) - if cycles then - cycles = cycles/iterations - io.write(('%.2f cycles, '):format(cycles)) - end - ns = ns/iterations - io.write(('%.2f ns per iteration (result: %s)\n'):format( - ns, tostring(res))) - if cycles and cycles > max_cycles then - print('WARNING: perfmark failed: exceeded maximum cycles '..max_cycles) - end - if ns > max_ns then - print('WARNING: perfmark failed: exceeded maximum ns '..max_ns) - end - return res - end - - local function test_jenkins(iterations) - local result - for i=1,iterations do result=hash_i32(i) end - return result - end - - local function test_murmur(iterations) - local result - for i=1,iterations do result=murmur_hash_i32(i) end - return result - end - - check_perf(test_jenkins, 1e8, 15, 4, 'jenkins hash') - check_perf(test_murmur, 1e8, 30, 8, 'murmur hash (32 bit)') - - -- 32-byte entries - local rhh = PodHashMap.new(ffi.typeof('uint32_t'), ffi.typeof('int32_t[6]'), - hash_i32) - local occupancy = 2e6 - rhh:resize(occupancy / 0.4 + 1) - - local function test_insertion(count) - local v = ffi.new('int32_t[6]'); - for i = 1,count do - for j=0,5 do v[j] = bnot(i) end - rhh:add(i, v) - end - end - - local function test_lookup(count) - local result - for i = 1, count do - result = rhh:lookup(i) - end - return result - end - - check_perf(test_insertion, occupancy, 400, 100, 'insertion (40% occupancy)') - print('max displacement: '..rhh.max_displacement) - io.write('selfcheck: ') - io.flush() - rhh:selfcheck() - io.write('pass\n') - - io.write('population check: ') - io.flush() - for i = 1, occupancy do - local offset = rhh:lookup(i) - assert(rhh:val_at(offset)[0] == bnot(i)) - end - rhh:selfcheck() - io.write('pass\n') - - io.write('iteration check: ') - io.flush() - local iterated = 0 - for entry in rhh:iterate() do iterated = iterated + 1 end - assert(iterated == occupancy) - io.write('pass\n') - - check_perf(test_lookup, occupancy, 300, 100, 'lookup (40% occupancy)') - - local stride = 1 - repeat - local streamer = rhh:make_lookup_streamer(stride) - local function test_lookup_streamer(count) - local result - for i = 1, count, stride do - local n = math.min(stride, count-i+1) - for j = 0, n-1 do - streamer.entries[j].key = i + j - end - streamer:stream() - result = streamer.entries[n-1].value[0] - end - return result - end - -- Note that "result" is part of the value, not an index into - -- the table, and so we expect the results to be different from - -- rhh:lookup(). - check_perf(test_lookup_streamer, occupancy, 1000, 100, - 'streaming lookup, stride='..stride) - stride = stride * 2 - until stride > 256 - - check_perf(test_lookup, occupancy, 300, 100, 'lookup (40% occupancy)') - - -- A check that our equality functions work as intended. - local numbers_equal = make_equal_fn(ffi.typeof('int')) - local four_byte = ffi.typeof('uint32_t[1]') - local eight_byte = ffi.typeof('uint32_t[2]') - local twelve_byte = ffi.typeof('uint32_t[3]') - local four_byte_equal = make_equal_fn(four_byte) - local eight_byte_equal = make_equal_fn(eight_byte) - local twelve_byte_equal = make_equal_fn(twelve_byte) - assert(numbers_equal(1,1)) - assert(not numbers_equal(1,2)) - assert(four_byte_equal(ffi.new(four_byte, {1}), - ffi.new(four_byte, {1}))) - assert(not four_byte_equal(ffi.new(four_byte, {1}), - ffi.new(four_byte, {2}))) - assert(eight_byte_equal(ffi.new(eight_byte, {1,1}), - ffi.new(eight_byte, {1,1}))) - assert(not eight_byte_equal(ffi.new(eight_byte, {1,1}), - ffi.new(eight_byte, {1,2}))) - assert(twelve_byte_equal(ffi.new(twelve_byte, {1,1,1}), - ffi.new(twelve_byte, {1,1,1}))) - assert(not twelve_byte_equal(ffi.new(twelve_byte, {1,1,1}), - ffi.new(twelve_byte, {1,1,2}))) -end diff --git a/src/apps/lwaftr/test_phm_create.lua b/src/apps/lwaftr/test_phm_create.lua deleted file mode 100644 index 9ba0acd265..0000000000 --- a/src/apps/lwaftr/test_phm_create.lua +++ /dev/null @@ -1,67 +0,0 @@ -local ffi = require('ffi') -local bit = require('bit') -local phm = require("apps.lwaftr.podhashmap") -local stream = require("apps.lwaftr.stream") - --- e.g. ./snabb snsh apps/lwaftr/test_phm_create.lua count occupancy filename -local function run(params) - if #params ~= 3 then error('usage: test_phm_create.lua COUNT OCCUPANCY FILENAME') end - local count, occupancy, filename = unpack(params) - count = assert(tonumber(count), "count not a number: "..count) - occupancy = assert(tonumber(occupancy), "occupancy not a number: "..occupancy) - assert(occupancy > 0 and occupancy < 1, - "occupancy should be between 0 and 1: "..occupancy) - - print(('creating uint32->int32[6] map with %d entries, %.0f%% occupancy'):format( - count, occupancy * 100)) - local key_t, value_t = ffi.typeof('uint32_t'), ffi.typeof('int32_t[6]') - local rhh = phm.PodHashMap.new(key_t, value_t, phm.hash_i32) - rhh:resize(math.ceil(count / occupancy)) - local start = ffi.C.get_time_ns() - do - local value = value_t() - for i = 1, count do - local v = bit.bnot(i) - value[0], value[1], value[2] = v, v, v - value[3], value[4], value[5] = v, v, v - rhh:add(i, value) - end - end - local stop = ffi.C.get_time_ns() - local ns = tonumber(stop-start)/count - print(ns..' ns/insertion') - - local max_displacement = rhh.max_displacement - print('max displacement: '..max_displacement) - print('saving '..filename) - local out = stream.open_temporary_output_byte_stream(filename) - rhh:save(out) - out:close_and_rename() - - print('reloading saved file') - rhh = phm.load(stream.open_input_byte_stream(filename), - key_t, value_t, phm.hash_i32) - - print('verifying saved file') - print('max displacement: '..rhh.max_displacement) - assert(rhh.max_displacement == max_displacement) - for i = 0, rhh.size + max_displacement do - local entry = rhh.entries[i] - if entry.hash ~= 0xffffffff then - assert(entry.hash == phm.hash_i32(entry.key)) - assert(entry.value[0] == bit.bnot(entry.key)) - assert(entry.value[5] == bit.bnot(entry.key)) - end - end - - for i = 1, count do - local offset = rhh:lookup(i) - assert(rhh:val_at(offset)[0] == bit.bnot(i)) - end - - -- rhh:dump() - - print("done") -end - -run(main.parameters) diff --git a/src/apps/lwaftr/test_phm_lookup.lua b/src/apps/lwaftr/test_phm_lookup.lua deleted file mode 100644 index fe436a33de..0000000000 --- a/src/apps/lwaftr/test_phm_lookup.lua +++ /dev/null @@ -1,40 +0,0 @@ -local ffi = require('ffi') -local phm = require("apps.lwaftr.podhashmap") -local stream = require("apps.lwaftr.stream") - -local function test(rhh, count, active) - print('lookup1 speed test (hits, uniform distribution)') - print(count..' lookups, '..(active or count)..' active keys') - local start = ffi.C.get_time_ns() - local result - for i = 1, count do - if active then i = (i % active) + 1 end - result = rhh:val_at(rhh:lookup(i))[0] - end - local stop = ffi.C.get_time_ns() - local ns = tonumber(stop-start)/count - print(ns..' ns/lookup (final result: '..result..')') -end - -local function run(params) - if #params < 1 or #params > 2 then - error('usage: test_phm_lookup.lua FILENAME [ACTIVE]') - end - local filename, active = unpack(params) - if active then - active = assert(tonumber(active), 'active should be a number') - assert(active == math.floor(active) and active > 0, - 'active should be a positive integer') - end - - local key_t, value_t = ffi.typeof('uint32_t'), ffi.typeof('int32_t[6]') - print('loading saved file '..filename) - local input = stream.open_input_byte_stream(filename) - local rhh = phm.load(input, key_t, value_t, phm.hash_i32) - - test(rhh, rhh.occupancy, active) - - print("done") -end - -run(main.parameters) diff --git a/src/apps/lwaftr/test_phm_streaming_lookup.lua b/src/apps/lwaftr/test_phm_streaming_lookup.lua deleted file mode 100644 index 86cee269c9..0000000000 --- a/src/apps/lwaftr/test_phm_streaming_lookup.lua +++ /dev/null @@ -1,60 +0,0 @@ -local ffi = require('ffi') -local phm = require("apps.lwaftr.podhashmap") -local stream = require("apps.lwaftr.stream") - -local function test(rhh, count, stride, active) - print('streaming lookup speed test (hits, uniform distribution)') - local streamer = rhh:make_lookup_streamer(stride) - print(count..' lookups, '..(active or count)..' active keys') - print('batching '..stride..' lookups at a time') - local start = ffi.C.get_time_ns() - local result - for i = 1, count, stride do - local n = math.min(stride, count + 1 - i) - for j = 0, n-1 do - if active then - streamer.entries[j].key = ((i+j) % active) + 1 - else - streamer.entries[j].key = i+j - end - end - streamer:stream() - result = streamer.entries[n-1].value[0] - end - local stop = ffi.C.get_time_ns() - local ns = tonumber(stop-start)/count - print(ns..' ns/lookup (final result: '..result..')') -end - --- e.g. ./snabb snsh apps/lwaftr/test_phm_streaming_lookup.lua foo.phm -local function run(params) - if #params < 1 or #params > 3 then - error('usage: test_phm_streaming_lookup.lua FILENAME [STRIDE] [ACTIVE]') - end - local filename, stride, active = unpack(params) - if stride then - stride = assert(tonumber(stride), 'stride should be a number') - assert(stride == math.floor(stride) and stride > 0, - 'stride should be a positive integer') - else - stride = 32 - end - if active then - active = assert(tonumber(active), 'active should be a number') - assert(active == math.floor(active) and active > 0, - 'active should be a positive integer') - end - - local key_t, value_t = ffi.typeof('uint32_t'), ffi.typeof('int32_t[6]') - print('loading saved file '..filename) - local input = stream.open_input_byte_stream(filename) - local rhh = phm.load(input, key_t, value_t, phm.hash_i32) - - print('max displacement: '..rhh.max_displacement) - - test(rhh, rhh.occupancy, stride, active) - - print("done") -end - -run(main.parameters) From 13488577381c7f96290553d48846fb0c40e76a51 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 14:36:11 +0100 Subject: [PATCH 172/631] Isolate old-style lwaftr configuration support Move this code to "snabb lwaftr migrate-configuration". --- src/apps/lwaftr/conf.lua | 193 +----------------- .../migrate_configuration.lua | 189 ++++++++++++++++- 2 files changed, 194 insertions(+), 188 deletions(-) diff --git a/src/apps/lwaftr/conf.lua b/src/apps/lwaftr/conf.lua index 21e694bf90..5ef40b8319 100644 --- a/src/apps/lwaftr/conf.lua +++ b/src/apps/lwaftr/conf.lua @@ -1,194 +1,15 @@ module(..., package.seeall) -local ethernet = require("lib.protocol.ethernet") -local ffi = require("ffi") -local ipv4 = require("lib.protocol.ipv4") -local ipv6 = require("lib.protocol.ipv6") -local util = require("lib.yang.util") local yang = require('lib.yang.yang') -local cltable = require('lib.cltable') -local Parser = require("apps.lwaftr.conf_parser").Parser - -policies = { - DROP = 1, - ALLOW = 2 -} - -local function required(key) - return function(config) - error('missing required configuration key "'..key..'"') - end -end - -local function required_if(key, otherkey) - return function(config) - if config[otherkey] then - error('missing required configuration key "'..key..'"') - end - end -end - -local function required_at_least_one_of(key, otherkey) - return function(config) - if config[otherkey] == nil then - error(string.format("At least one of '%s' and '%s' must be specified", key, otherkey)) - end - end -end - -local function default(val) - return function(config) return val end -end - -local lwaftr_conf_spec = { - parse={ - aftr_ipv4_ip=Parser.parse_ipv4, - aftr_ipv6_ip=Parser.parse_ipv6, - aftr_mac_b4_side=Parser.parse_mac, - aftr_mac_inet_side=Parser.parse_mac, - next_hop6_mac=Parser.parse_mac, - binding_table=Parser.parse_file_name, - hairpinning=Parser.parse_boolean, - icmpv4_rate_limiter_n_packets=Parser.parse_non_negative_number, - icmpv4_rate_limiter_n_seconds=Parser.parse_positive_number, - icmpv6_rate_limiter_n_packets=Parser.parse_non_negative_number, - icmpv6_rate_limiter_n_seconds=Parser.parse_positive_number, - inet_mac=Parser.parse_mac, - ipv4_mtu=Parser.parse_mtu, - ipv6_mtu=Parser.parse_mtu, - max_fragments_per_reassembly_packet=Parser.parse_positive_number, - max_ipv4_reassembly_packets=Parser.parse_positive_number, - max_ipv6_reassembly_packets=Parser.parse_positive_number, - next_hop_ipv4_addr=Parser.parse_ipv4, - next_hop_ipv6_addr=Parser.parse_ipv6, - policy_icmpv4_incoming=Parser.enum_parser(policies), - policy_icmpv4_outgoing=Parser.enum_parser(policies), - policy_icmpv6_incoming=Parser.enum_parser(policies), - policy_icmpv6_outgoing=Parser.enum_parser(policies), - v4_vlan_tag=Parser.parse_vlan_tag, - v6_vlan_tag=Parser.parse_vlan_tag, - vlan_tagging=Parser.parse_boolean, - ipv4_ingress_filter=Parser.parse_string_or_file, - ipv4_egress_filter=Parser.parse_string_or_file, - ipv6_ingress_filter=Parser.parse_string_or_file, - ipv6_egress_filter=Parser.parse_string_or_file, - }, - defaults={ - aftr_ipv4_ip=required('aftr_ipv4_ip'), - aftr_ipv6_ip=required('aftr_ipv6_ip'), - aftr_mac_b4_side=required('aftr_mac_b4_side'), - aftr_mac_inet_side=required('aftr_mac_inet_side'), - next_hop6_mac=required_at_least_one_of('next_hop6_mac', 'next_hop_ipv6_addr'), - binding_table=required('binding_table'), - hairpinning=default(true), - icmpv4_rate_limiter_n_packets=default(6e5), - icmpv4_rate_limiter_n_seconds=default(2), - icmpv6_rate_limiter_n_packets=default(6e5), - icmpv6_rate_limiter_n_seconds=default(2), - inet_mac=required_at_least_one_of('inet_mac', 'next_hop_ipv4_addr'), - ipv4_mtu=default(1460), - ipv6_mtu=default(1500), - max_fragments_per_reassembly_packet=default(40), - max_ipv4_reassembly_packets=default(20000), -- Just under 500 megs memory - max_ipv6_reassembly_packets=default(20000), -- Just under 500 megs memory - next_hop_ipv4_addr = required_at_least_one_of('next_hop_ipv4_addr', 'inet_mac'), - next_hop_ipv6_addr = required_at_least_one_of('next_hop_ipv6_addr', 'next_hop6_mac'), - policy_icmpv4_incoming=default(policies.ALLOW), - policy_icmpv4_outgoing=default(policies.ALLOW), - policy_icmpv6_incoming=default(policies.ALLOW), - policy_icmpv6_outgoing=default(policies.ALLOW), - v4_vlan_tag=required_if('v4_vlan_tag', 'vlan_tagging'), - v6_vlan_tag=required_if('v6_vlan_tag', 'vlan_tagging'), - vlan_tagging=default(false) - }, - validate=function(parser, config) end -} - -function migrate_conf(old) - function convert_ipv4(addr) - if addr then return util.ipv4_pton(ipv4:ntop(addr)) end - end - local external = { - ip = convert_ipv4(old.aftr_ipv4_ip), - mac = old.aftr_mac_inet_side, - mtu = old.ipv4_mtu, - ingress_filter = old.ipv4_ingress_filter, - egress_filter = old.ipv4_egress_filter, - allow_incoming_icmp = old.policy_icmpv4_incoming == policies.ALLOW, - generate_icmp_errors = old.policy_icmpv4_outgoing == policies.ALLOW, - vlan_tag = old.v4_vlan_tag, - error_rate_limiting = { - packets = old.icmpv4_rate_limiter_n_packets, - period = old.icmpv4_rate_limiter_n_seconds - }, - reassembly = { - max_fragments_per_packet = old.max_fragments_per_reassembly_packet, - max_packets = old.max_ipv4_reassembly_packets - }, - next_hop = { - ip = convert_ipv4(old.next_hop_ipv4_addr), - mac = old.inet_mac - } - } - - local internal = { - ip = old.aftr_ipv6_ip, - mac = old.aftr_mac_b4_side, - mtu = old.ipv6_mtu, - ingress_filter = old.ipv6_ingress_filter, - egress_filter = old.ipv6_egress_filter, - allow_incoming_icmp = old.policy_icmpv6_incoming == policies.ALLOW, - generate_icmp_errors = old.policy_icmpv6_outgoing == policies.ALLOW, - vlan_tag = old.v6_vlan_tag, - error_rate_limiting = { - packets = old.icmpv6_rate_limiter_n_packets, - period = old.icmpv6_rate_limiter_n_seconds - }, - reassembly = { - max_fragments_per_packet = old.max_fragments_per_reassembly_packet, - max_packets = old.max_ipv6_reassembly_packets - }, - next_hop = { - ip = old.next_hop_ipv6_addr, - mac = old.next_hop6_mac - }, - hairpinning = old.hairpinning - } - - local binding_table = require("apps.lwaftr.binding_table") - local old_bt = binding_table.load_legacy(old.binding_table) - local psid_key_t = ffi.typeof('struct { uint32_t addr; }') - local psid_map = cltable.new({ key_type = psid_key_t }) - for addr, end_addr, params in old_bt.psid_map:iterate() do - local reserved_ports_bit_count = 16 - params.psid_length - params.shift - if end_addr == addr then end_addr = nil end - if reserved_ports_bit_count ~= 16 then - psid_map[psid_key_t(addr)] = { - end_addr = end_addr, - psid_length = params.psid_length, - shift = params.shift, - reserved_ports_bit_count = reserved_ports_bit_count - } - end - end - - return { - external_interface = external, - internal_interface = internal, - binding_table = { - psid_map = psid_map, - br_address = old_bt.br_addresses, - softwire = old_bt.softwires - } - } -end - -function load_legacy_lwaftr_config(stream) - local conf = Parser.new(stream):parse_property_list(lwaftr_conf_spec) - return migrate_conf(conf) -end function load_lwaftr_config(filename) + -- FIXME: apply constraints on config: + -- next_hop6_mac=required_at_least_one_of('next_hop6_mac', 'next_hop_ipv6_addr'), + -- inet_mac=required_at_least_one_of('inet_mac', 'next_hop_ipv4_addr'), + -- next_hop_ipv4_addr = required_at_least_one_of('next_hop_ipv4_addr', 'inet_mac'), + -- next_hop_ipv6_addr = required_at_least_one_of('next_hop_ipv6_addr', 'next_hop6_mac'), + -- v4_vlan_tag=required_if('v4_vlan_tag', 'vlan_tagging'), + -- v6_vlan_tag=required_if('v6_vlan_tag', 'vlan_tagging'), return yang.load_configuration(filename, {schema_name='snabb-softwire-v1'}) end diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 3a3b71ef7b..09376904b6 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -1,9 +1,15 @@ module(..., package.seeall) local lib = require('core.lib') -local conf = require('apps.lwaftr.conf') -local load_legacy_lwaftr_config = conf.load_legacy_lwaftr_config +local ffi = require("ffi") +local ethernet = require("lib.protocol.ethernet") +local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") +local cltable = require('lib.cltable') +local util = require('lib.yang.util') local yang = require('lib.yang.yang') +local Parser = require("apps.lwaftr.conf_parser").Parser +local binding_table = require("apps.lwaftr.binding_table") local function show_usage(code) print(require("program.lwaftr.migrate_configuration.README_inc")) @@ -18,6 +24,185 @@ local function parse_args(args) return unpack(args) end +local policies = { + DROP = 1, + ALLOW = 2 +} + +local function required(key) + return function(config) + error('missing required configuration key "'..key..'"') + end +end + +local function required_if(key, otherkey) + return function(config) + if config[otherkey] then + error('missing required configuration key "'..key..'"') + end + end +end + +local function required_at_least_one_of(key, otherkey) + return function(config) + if config[otherkey] == nil then + error(string.format("At least one of '%s' and '%s' must be specified", key, otherkey)) + end + end +end + +local function default(val) + return function(config) return val end +end + +local lwaftr_conf_spec = { + parse={ + aftr_ipv4_ip=Parser.parse_ipv4, + aftr_ipv6_ip=Parser.parse_ipv6, + aftr_mac_b4_side=Parser.parse_mac, + aftr_mac_inet_side=Parser.parse_mac, + next_hop6_mac=Parser.parse_mac, + binding_table=Parser.parse_file_name, + hairpinning=Parser.parse_boolean, + icmpv4_rate_limiter_n_packets=Parser.parse_non_negative_number, + icmpv4_rate_limiter_n_seconds=Parser.parse_positive_number, + icmpv6_rate_limiter_n_packets=Parser.parse_non_negative_number, + icmpv6_rate_limiter_n_seconds=Parser.parse_positive_number, + inet_mac=Parser.parse_mac, + ipv4_mtu=Parser.parse_mtu, + ipv6_mtu=Parser.parse_mtu, + max_fragments_per_reassembly_packet=Parser.parse_positive_number, + max_ipv4_reassembly_packets=Parser.parse_positive_number, + max_ipv6_reassembly_packets=Parser.parse_positive_number, + next_hop_ipv4_addr=Parser.parse_ipv4, + next_hop_ipv6_addr=Parser.parse_ipv6, + policy_icmpv4_incoming=Parser.enum_parser(policies), + policy_icmpv4_outgoing=Parser.enum_parser(policies), + policy_icmpv6_incoming=Parser.enum_parser(policies), + policy_icmpv6_outgoing=Parser.enum_parser(policies), + v4_vlan_tag=Parser.parse_vlan_tag, + v6_vlan_tag=Parser.parse_vlan_tag, + vlan_tagging=Parser.parse_boolean, + ipv4_ingress_filter=Parser.parse_string_or_file, + ipv4_egress_filter=Parser.parse_string_or_file, + ipv6_ingress_filter=Parser.parse_string_or_file, + ipv6_egress_filter=Parser.parse_string_or_file, + }, + defaults={ + aftr_ipv4_ip=required('aftr_ipv4_ip'), + aftr_ipv6_ip=required('aftr_ipv6_ip'), + aftr_mac_b4_side=required('aftr_mac_b4_side'), + aftr_mac_inet_side=required('aftr_mac_inet_side'), + next_hop6_mac=required_at_least_one_of('next_hop6_mac', 'next_hop_ipv6_addr'), + binding_table=required('binding_table'), + hairpinning=default(true), + icmpv4_rate_limiter_n_packets=default(6e5), + icmpv4_rate_limiter_n_seconds=default(2), + icmpv6_rate_limiter_n_packets=default(6e5), + icmpv6_rate_limiter_n_seconds=default(2), + inet_mac=required_at_least_one_of('inet_mac', 'next_hop_ipv4_addr'), + ipv4_mtu=default(1460), + ipv6_mtu=default(1500), + max_fragments_per_reassembly_packet=default(40), + max_ipv4_reassembly_packets=default(20000), -- Just under 500 megs memory + max_ipv6_reassembly_packets=default(20000), -- Just under 500 megs memory + next_hop_ipv4_addr = required_at_least_one_of('next_hop_ipv4_addr', 'inet_mac'), + next_hop_ipv6_addr = required_at_least_one_of('next_hop_ipv6_addr', 'next_hop6_mac'), + policy_icmpv4_incoming=default(policies.ALLOW), + policy_icmpv4_outgoing=default(policies.ALLOW), + policy_icmpv6_incoming=default(policies.ALLOW), + policy_icmpv6_outgoing=default(policies.ALLOW), + v4_vlan_tag=required_if('v4_vlan_tag', 'vlan_tagging'), + v6_vlan_tag=required_if('v6_vlan_tag', 'vlan_tagging'), + vlan_tagging=default(false) + }, + validate=function(parser, config) end +} + + +local function migrate_conf(old) + function convert_ipv4(addr) + if addr then return util.ipv4_pton(ipv4:ntop(addr)) end + end + local external = { + ip = convert_ipv4(old.aftr_ipv4_ip), + mac = old.aftr_mac_inet_side, + mtu = old.ipv4_mtu, + ingress_filter = old.ipv4_ingress_filter, + egress_filter = old.ipv4_egress_filter, + allow_incoming_icmp = old.policy_icmpv4_incoming == policies.ALLOW, + generate_icmp_errors = old.policy_icmpv4_outgoing == policies.ALLOW, + vlan_tag = old.v4_vlan_tag, + error_rate_limiting = { + packets = old.icmpv4_rate_limiter_n_packets, + period = old.icmpv4_rate_limiter_n_seconds + }, + reassembly = { + max_fragments_per_packet = old.max_fragments_per_reassembly_packet, + max_packets = old.max_ipv4_reassembly_packets + }, + next_hop = { + ip = convert_ipv4(old.next_hop_ipv4_addr), + mac = old.inet_mac + } + } + + local internal = { + ip = old.aftr_ipv6_ip, + mac = old.aftr_mac_b4_side, + mtu = old.ipv6_mtu, + ingress_filter = old.ipv6_ingress_filter, + egress_filter = old.ipv6_egress_filter, + allow_incoming_icmp = old.policy_icmpv6_incoming == policies.ALLOW, + generate_icmp_errors = old.policy_icmpv6_outgoing == policies.ALLOW, + vlan_tag = old.v6_vlan_tag, + error_rate_limiting = { + packets = old.icmpv6_rate_limiter_n_packets, + period = old.icmpv6_rate_limiter_n_seconds + }, + reassembly = { + max_fragments_per_packet = old.max_fragments_per_reassembly_packet, + max_packets = old.max_ipv6_reassembly_packets + }, + next_hop = { + ip = old.next_hop_ipv6_addr, + mac = old.next_hop6_mac + }, + hairpinning = old.hairpinning + } + + local old_bt = binding_table.load_legacy(old.binding_table) + local psid_key_t = ffi.typeof('struct { uint32_t addr; }') + local psid_map = cltable.new({ key_type = psid_key_t }) + for addr, end_addr, params in old_bt.psid_map:iterate() do + local reserved_ports_bit_count = 16 - params.psid_length - params.shift + if end_addr == addr then end_addr = nil end + if reserved_ports_bit_count ~= 16 then + psid_map[psid_key_t(addr)] = { + end_addr = end_addr, + psid_length = params.psid_length, + shift = params.shift, + reserved_ports_bit_count = reserved_ports_bit_count + } + end + end + + return { + external_interface = external, + internal_interface = internal, + binding_table = { + psid_map = psid_map, + br_address = old_bt.br_addresses, + softwire = old_bt.softwires + } + } +end + +local function load_legacy_lwaftr_config(stream) + local conf = Parser.new(stream):parse_property_list(lwaftr_conf_spec) + return migrate_conf(conf) +end + function run(args) local conf_file = parse_args(args) local conf = load_legacy_lwaftr_config(conf_file) From f916dea4ad1ddaf2f5b30c5555ff5fcd0986f62c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 15:33:37 +0100 Subject: [PATCH 173/631] Fix default value bug in yang parser * src/lib/yang/data.lua: Fix bug parsing default values. --- src/lib/yang/data.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 893e4df645..4182d9a83f 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -212,7 +212,7 @@ local function scalar_parser(keyword, argument_type, default, mandatory) end local function finish(out) if out ~= nil then return out end - if default then return parse1(default) end + if default then return parsev(default) end if mandatory then error('missing scalar value: '..keyword) end end return {init=init, parse=parse, finish=finish} From fd658106bcddb1a1772b6dd4871fcb793f4eedbc Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 15:33:56 +0100 Subject: [PATCH 174/631] snabb-softwire-v1 tweak * src/lib/yang/snabb-softwire-v1.yang: Br defaults to 0. --- src/lib/yang/snabb-softwire-v1.yang | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 238ebee337..ab9046816e 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -363,7 +363,7 @@ module snabb-softwire-v1 { leaf br { type uint32; - mandatory true; + default 0; description "The B4-facing address of the lwAFTR for this softwire, as a zero-based index into br-addresses."; From b7a57741e72bf8a883433aaa118af68d9c3a42f7 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 15:46:13 +0100 Subject: [PATCH 175/631] Omit default values in yang serialization --- src/lib/yang/data.lua | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 4182d9a83f..47af891adf 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -364,16 +364,16 @@ local function encode_yang_string(str) return table.concat(out) end -local value_printers = {} -local function value_printer(typ) +local value_serializers = {} +local function value_serializer(typ) local prim = typ.primitive_type - if value_printers[prim] then return value_printers[prim] end + if value_serializers[prim] then return value_serializers[prim] end local tostring = assert(value.types[prim], prim).tostring - local function print(val, file) - file:write(encode_yang_string(tostring(val))) + local function serializer(val) + return encode_yang_string(tostring(val)) end - value_printers[prim] = print - return print + value_serializers[prim] = serializer + return serializer end local function data_printer_from_grammar(production) @@ -416,11 +416,11 @@ local function data_printer_from_grammar(production) end end function handlers.array(keyword, production) - local print_value = value_printer(production.element_type) + local serialize = value_serializer(production.element_type) return function(data, file, indent) for _,v in ipairs(data) do print_keyword(keyword, file, indent) - print_value(v, file) + file:write(serialize(data)) file:write(';\n') end end @@ -477,11 +477,14 @@ local function data_printer_from_grammar(production) end end function handlers.scalar(keyword, production) - local print_value = value_printer(production.argument_type) + local serialize = value_serializer(production.argument_type) return function(data, file, indent) - print_keyword(keyword, file, indent) - print_value(data, file) - file:write(';\n') + local str = serialize(data) + if str ~= production.default then + print_keyword(keyword, file, indent) + file:write(str) + file:write(';\n') + end end end From ce3ecc365227100585f0027a67f103beadb86819 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 15:46:45 +0100 Subject: [PATCH 176/631] Move more legacy code to migrate-configuration --- src/apps/lwaftr/binding_table.lua | 241 +++--------------- .../migrate_configuration.lua | 144 ++++++++++- 2 files changed, 182 insertions(+), 203 deletions(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index fbc02390d7..9f3c00ef7b 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -65,39 +65,22 @@ module(..., package.seeall) local bit = require('bit') local ffi = require("ffi") -local stream = require("apps.lwaftr.stream") -local lwdebug = require("apps.lwaftr.lwdebug") -local Parser = require("apps.lwaftr.conf_parser").Parser local rangemap = require("apps.lwaftr.rangemap") -local ctable = require("lib.ctable") local cltable = require("lib.cltable") -local util = require("lib.yang.util") local band, lshift, rshift = bit.band, bit.lshift, bit.rshift -local psid_map_value_t = ffi.typeof[[ +-- FIXME: Pull these types from the yang model, not out of thin air. +psid_map_value_t = ffi.typeof[[ struct { uint16_t psid_length; uint16_t shift; } ]] - -local br_address_t = ffi.typeof('uint8_t[16]') - --- Total softwire entry size is 32 bytes (with the 4 byte hash), which --- has nice cache alignment properties. -local softwire_key_t = ffi.typeof[[ +softwire_key_t = ffi.typeof[[ struct { uint32_t ipv4; // Public IPv4 address of this softwire (host-endian). - uint16_t psid; // Port set ID. uint16_t padding; // Zeroes. + uint16_t psid; // Port set ID. } __attribute__((packed)) ]] -local softwire_value_t = ffi.typeof[[ - struct { - uint32_t br; // Which border router (lwAFTR IPv6 address)? - uint8_t b4_ipv6[16]; // Address of B4. - } __attribute__((packed)) -]] - -local SOFTWIRE_TABLE_LOAD_FACTOR = 0.4 BTLookupQueue = {} @@ -253,137 +236,6 @@ function BindingTable:iterate_softwires() return self.softwires:iterate() end -local function parse_psid_map(parser) - local psid_info_spec = { - parse={ - psid_length=Parser.parse_psid_param, - shift=Parser.parse_psid_param - }, - defaults={ - psid_length=function(config) return 16 - (config.shift or 16) end, - shift=function(config) return 16 - (config.psid_length or 0) end - }, - validate=function(parser, config) - if config.psid_length + config.shift ~= 16 then - parser:error('psid_length %d + shift %d should add up to 16', - config.psid_length, config.shift) - end - end - } - - local builder = rangemap.RangeMapBuilder.new(psid_map_value_t) - local value = psid_map_value_t() - parser:skip_whitespace() - parser:consume_token('[%a_]', 'psid_map') - parser:skip_whitespace() - parser:consume('{') - parser:skip_whitespace() - while not parser:check('}') do - local range_list = parser:parse_ipv4_range_list() - local info = parser:parse_property_list(psid_info_spec, '{', '}') - value.psid_length, value.shift = info.psid_length, info.shift - for _, range in ipairs(range_list) do - builder:add_range(range.min, range.max, value) - end - parser:skip_whitespace() - if parser:check(',') or parser:check(';') then - parser:skip_whitespace() - end - end - return builder:build(psid_map_value_t()) -end - -local function parse_br_addresses(parser) - local addresses = {} - parser:skip_whitespace() - parser:consume_token('[%a_]', 'br_addresses') - parser:skip_whitespace() - parser:consume('{') - parser:skip_whitespace() - while not parser:check('}') do - table.insert(addresses, parser:parse_ipv6()) - parser:skip_whitespace() - if parser:check(',') then parser:skip_whitespace() end - end - local ret = util.ffi_array(ffi.new(ffi.typeof('$[?]', br_address_t), - #addresses), - br_address_t, #addresses) - for i, addr in ipairs(addresses) do ret[i] = addr end - return ret -end - -local function parse_softwires(parser, psid_map, br_address_count) - local function required(key) - return function(config) - error('missing required configuration key "'..key..'"') - end - end - local softwire_spec = { - parse={ - ipv4=Parser.parse_ipv4_as_uint32, - psid=Parser.parse_psid, - b4=Parser.parse_ipv6, - aftr=Parser.parse_non_negative_number - }, - defaults={ - ipv4=required('ipv4'), - psid=function(config) return 0 end, - b4=required('b4'), - aftr=function(config) return 0 end - }, - validate=function(parser, config) - local psid_length = psid_map:lookup(config.ipv4).value.psid_length - if config.psid >= 2^psid_length then - parser:error('psid %d out of range for IP', config.psid) - end - if config.aftr >= br_address_count then - parser:error('only %d br addresses are defined', br_address_count) - end - end - } - - local map = ctable.new( - { key_type = softwire_key_t, value_type = softwire_value_t }) - local key, value = softwire_key_t(), softwire_value_t() - parser:skip_whitespace() - parser:consume_token('[%a_]', 'softwires') - parser:skip_whitespace() - parser:consume('{') - parser:skip_whitespace() - while not parser:check('}') do - local entry = parser:parse_property_list(softwire_spec, '{', '}') - key.ipv4, key.psid = entry.ipv4, entry.psid - value.br, value.b4_ipv6 = entry.aftr, entry.b4 - local success = pcall(map.add, map, key, value) - if not success then - parser:error('duplicate softwire for ipv4=%s, psid=%d', - lwdebug.format_ipv4(key.ipv4), key.psid) - end - parser:skip_whitespace() - if parser:check(',') then parser:skip_whitespace() end - end - map:resize(map.size / SOFTWIRE_TABLE_LOAD_FACTOR) - return map -end - -local function parse_binding_table(parser) - local psid_map = parse_psid_map(parser) - local br_addresses = parse_br_addresses(parser) - local softwires = parse_softwires(parser, psid_map, #br_addresses) - parser:skip_whitespace() - parser:consume(nil) - return BindingTable.new(psid_map, br_addresses, softwires) -end - -function load_source(text_stream) - return parse_binding_table(Parser.new(text_stream)) -end - -function load_legacy(file) - local source = stream.open_input_byte_stream(file) - return load_source(source:as_text_stream()) -end - function load(conf) local psid_builder = rangemap.RangeMapBuilder.new(psid_map_value_t) local psid_value = psid_map_value_t() @@ -402,55 +254,40 @@ end function selftest() print('selftest: binding_table') - local function string_file(str) - local pos = 1 - return { - read = function(self, n) - assert(n==1) - local ret - if pos <= #str then - ret = str:sub(pos,pos) - pos = pos + 1 - end - return ret - end, - close = function(self) str = nil end - } + local function load_str(str) + local yang = require('lib.yang.yang') + local data = require('lib.yang.data') + local schema = yang.load_schema_by_name('snabb-softwire-v1') + local grammar = data.data_grammar_from_schema(schema) + local subgrammar = assert(grammar.members['binding-table']) + local parse = data.data_parser_from_grammar(subgrammar) + return load(parse(str, '[test suite]')) end - local map = load_source(string_file([[ - psid_map { - 178.79.150.233 {psid_length=16} - 178.79.150.15 {psid_length=4, shift=12} - 178.79.150.2 {psid_length=16} - 178.79.150.3 {psid_length=6} - } - br_addresses { - 8:9:a:b:c:d:e:f, - 1E:1:1:1:1:1:1:af, - 1E:2:2:2:2:2:2:af - } - softwires { - { ipv4=178.79.150.233, psid=80, b4=127:2:3:4:5:6:7:128, aftr=0 } - { ipv4=178.79.150.233, psid=2300, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=2700, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=4660, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=7850, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=22788, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=54192, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.15, psid=0, b4=127:22:33:44:55:66:77:128 } - { ipv4=178.79.150.15, psid=1, b4=127:22:33:44:55:66:77:128 } - { ipv4=178.79.150.2, psid=7850, b4=127:24:35:46:57:68:79:128, aftr=1 } - { ipv4=178.79.150.3, psid=4, b4=127:14:25:36:47:58:69:128, aftr=2 } - } - ]])) - - local ipv4_protocol = require("lib.protocol.ipv4") + local map = load_str([[ + psid-map { addr 178.79.150.233; psid-length 16; } + psid-map { addr 178.79.150.15; psid-length 4; shift 12; } + psid-map { addr 178.79.150.2; psid-length 16; } + psid-map { addr 178.79.150.3; psid-length 6; } + br-address 8:9:a:b:c:d:e:f; + br-address 1E:1:1:1:1:1:1:af; + br-address 1E:2:2:2:2:2:2:af; + softwire { ipv4 178.79.150.233; psid 80; b4-ipv6 127:2:3:4:5:6:7:128; br 0; } + softwire { ipv4 178.79.150.233; psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; } + softwire { ipv4 178.79.150.233; psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; } + softwire { ipv4 178.79.150.233; psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; } + softwire { ipv4 178.79.150.233; psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; } + softwire { ipv4 178.79.150.233; psid 22788; b4-ipv6 127:11:12:13:14:15:16:128; } + softwire { ipv4 178.79.150.233; psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; } + softwire { ipv4 178.79.150.15; psid 0; b4-ipv6 127:22:33:44:55:66:77:128; } + softwire { ipv4 178.79.150.15; psid 1; b4-ipv6 127:22:33:44:55:66:77:128;} + softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; br 1; } + softwire { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; br 2; } + ]]) + + local ipv4_pton = require('lib.yang.util').ipv4_pton local ipv6_protocol = require("lib.protocol.ipv6") - local function pton_host_uint32(ipv4) - return ffi.C.ntohl(ffi.cast('uint32_t*', ipv4_protocol:pton(ipv4))[0]) - end local function lookup(ipv4, port) - return map:lookup(pton_host_uint32(ipv4), port) + return map:lookup(ipv4_pton(ipv4), port) end local function assert_lookup(ipv4, port, ipv6, br) local val = assert(lookup(ipv4, port)) @@ -474,10 +311,10 @@ function selftest() do local psid_map_iter = { - { pton_host_uint32('178.79.150.2'), { psid_length=16, shift=0 } }, - { pton_host_uint32('178.79.150.3'), { psid_length=6, shift=10 } }, - { pton_host_uint32('178.79.150.15'), { psid_length=4, shift=12 } }, - { pton_host_uint32('178.79.150.233'), { psid_length=16, shift=0 } } + { ipv4_pton('178.79.150.2'), { psid_length=16, shift=0 } }, + { ipv4_pton('178.79.150.3'), { psid_length=6, shift=10 } }, + { ipv4_pton('178.79.150.15'), { psid_length=4, shift=12 } }, + { ipv4_pton('178.79.150.233'), { psid_length=16, shift=0 } } } local i = 1 for lo, hi, value in map:iterate_psid_map() do diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 09376904b6..8b979bee15 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -5,12 +5,18 @@ local ffi = require("ffi") local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") +local rangemap = require("apps.lwaftr.rangemap") +local ctable = require("lib.ctable") local cltable = require('lib.cltable') local util = require('lib.yang.util') local yang = require('lib.yang.yang') +local stream = require('lib.yang.stream') local Parser = require("apps.lwaftr.conf_parser").Parser local binding_table = require("apps.lwaftr.binding_table") +local br_address_t = ffi.typeof('uint8_t[16]') +local SOFTWIRE_TABLE_LOAD_FACTOR = 0.4 + local function show_usage(code) print(require("program.lwaftr.migrate_configuration.README_inc")) main.exit(code) @@ -119,6 +125,142 @@ local lwaftr_conf_spec = { validate=function(parser, config) end } +local function parse_psid_map(parser) + local psid_info_spec = { + parse={ + psid_length=Parser.parse_psid_param, + shift=Parser.parse_psid_param + }, + defaults={ + psid_length=function(config) return 16 - (config.shift or 16) end, + shift=function(config) return 16 - (config.psid_length or 0) end + }, + validate=function(parser, config) + if config.psid_length + config.shift > 16 then + parser:error('psid_length %d + shift %d should not exceed 16', + config.psid_length, config.shift) + end + end + } + + local builder = rangemap.RangeMapBuilder.new(binding_table.psid_map_value_t) + local value = binding_table.psid_map_value_t() + parser:skip_whitespace() + parser:consume_token('[%a_]', 'psid_map') + parser:skip_whitespace() + parser:consume('{') + parser:skip_whitespace() + while not parser:check('}') do + local range_list = parser:parse_ipv4_range_list() + local info = parser:parse_property_list(psid_info_spec, '{', '}') + value.psid_length, value.shift = info.psid_length, info.shift + for _, range in ipairs(range_list) do + builder:add_range(range.min, range.max, value) + end + parser:skip_whitespace() + if parser:check(',') or parser:check(';') then + parser:skip_whitespace() + end + end + return builder:build(binding_table.psid_map_value_t()) +end + +local function parse_br_addresses(parser) + local addresses = {} + parser:skip_whitespace() + parser:consume_token('[%a_]', 'br_addresses') + parser:skip_whitespace() + parser:consume('{') + parser:skip_whitespace() + while not parser:check('}') do + table.insert(addresses, parser:parse_ipv6()) + parser:skip_whitespace() + if parser:check(',') then parser:skip_whitespace() end + end + local ret = util.ffi_array(ffi.new(ffi.typeof('$[?]', br_address_t), + #addresses), + br_address_t, #addresses) + for i, addr in ipairs(addresses) do ret[i] = addr end + return ret +end + +local function parse_softwires(parser, psid_map, br_address_count) + local function required(key) + return function(config) + error('missing required configuration key "'..key..'"') + end + end + local softwire_spec = { + parse={ + ipv4=Parser.parse_ipv4_as_uint32, + psid=Parser.parse_psid, + b4=Parser.parse_ipv6, + aftr=Parser.parse_non_negative_number + }, + defaults={ + ipv4=required('ipv4'), + psid=function(config) return 0 end, + b4=required('b4'), + aftr=function(config) return 0 end + }, + validate=function(parser, config) + local psid_length = psid_map:lookup(config.ipv4).value.psid_length + if config.psid >= 2^psid_length then + parser:error('psid %d out of range for IP', config.psid) + end + if config.aftr >= br_address_count then + parser:error('only %d br addresses are defined', br_address_count) + end + end + } + + local softwire_key_t = binding_table.softwire_key_t + -- FIXME: Pull this type from the yang model, not out of thin air. + local softwire_value_t = ffi.typeof[[ + struct { + uint8_t b4_ipv6[16]; // Address of B4. + uint32_t br; // Which border router (lwAFTR IPv6 address)? + } __attribute__((packed)) + ]] + local map = ctable.new( + { key_type = softwire_key_t, value_type = softwire_value_t }) + local key, value = softwire_key_t(), softwire_value_t() + parser:skip_whitespace() + parser:consume_token('[%a_]', 'softwires') + parser:skip_whitespace() + parser:consume('{') + parser:skip_whitespace() + while not parser:check('}') do + local entry = parser:parse_property_list(softwire_spec, '{', '}') + key.ipv4, key.psid = entry.ipv4, entry.psid + value.br, value.b4_ipv6 = entry.aftr, entry.b4 + local success = pcall(map.add, map, key, value) + if not success then + parser:error('duplicate softwire for ipv4=%s, psid=%d', + lwdebug.format_ipv4(key.ipv4), key.psid) + end + parser:skip_whitespace() + if parser:check(',') then parser:skip_whitespace() end + end + map:resize(map.size / SOFTWIRE_TABLE_LOAD_FACTOR) + return map +end + +local function parse_binding_table(parser) + local psid_map = parse_psid_map(parser) + local br_addresses = parse_br_addresses(parser) + local softwires = parse_softwires(parser, psid_map, #br_addresses) + parser:skip_whitespace() + parser:consume(nil) + return { psid_map = psid_map, + br_addresses = br_addresses, + softwires = softwires } +end + +function load_binding_table(file) + local source = stream.open_input_byte_stream(file) + return parse_binding_table(Parser.new(source:as_text_stream())) +end local function migrate_conf(old) function convert_ipv4(addr) @@ -171,7 +313,7 @@ local function migrate_conf(old) hairpinning = old.hairpinning } - local old_bt = binding_table.load_legacy(old.binding_table) + local old_bt = load_binding_table(old.binding_table) local psid_key_t = ffi.typeof('struct { uint32_t addr; }') local psid_map = cltable.new({ key_type = psid_key_t }) for addr, end_addr, params in old_bt.psid_map:iterate() do From 0a92b692d997868d53c779a7e1c260a2c164c11e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 15:47:06 +0100 Subject: [PATCH 177/631] Remove old binding tables --- .../lwaftr/tests/data/binding-table.txt | 26 --- .../lwaftr/tests/data/vlan/binding-table.txt | 1 - .../snabbvmx/tests/conf/binding_table.txt | 199 ------------------ 3 files changed, 226 deletions(-) delete mode 100644 src/program/lwaftr/tests/data/binding-table.txt delete mode 120000 src/program/lwaftr/tests/data/vlan/binding-table.txt delete mode 100644 src/program/snabbvmx/tests/conf/binding_table.txt diff --git a/src/program/lwaftr/tests/data/binding-table.txt b/src/program/lwaftr/tests/data/binding-table.txt deleted file mode 100644 index 21903a5512..0000000000 --- a/src/program/lwaftr/tests/data/binding-table.txt +++ /dev/null @@ -1,26 +0,0 @@ -psid_map { - 178.79.150.1 {psid_length=0} - 178.79.150.2 {psid_length=16} - 178.79.150.3 {psid_length=6} - 178.79.150.15 {psid_length=4, shift=12} - 178.79.150.233 {psid_length=16} -} -br_addresses { - 8:9:a:b:c:d:e:f, - 1E:1:1:1:1:1:1:af, - 1E:2:2:2:2:2:2:af -} -softwires { - { ipv4=178.79.150.1, b4=127:10:20:30:40:50:60:128 } - { ipv4=178.79.150.2, psid=7850, b4=127:24:35:46:57:68:79:128, aftr=1 } - { ipv4=178.79.150.3, psid=4, b4=127:14:25:36:47:58:69:128, aftr=2 } - { ipv4=178.79.150.15, psid=0, b4=127:22:33:44:55:66:77:128 } - { ipv4=178.79.150.15, psid=1, b4=127:22:33:44:55:66:77:128 } - { ipv4=178.79.150.233, psid=80, b4=127:2:3:4:5:6:7:128, aftr=0 } - { ipv4=178.79.150.233, psid=2300, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=2700, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=4660, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=7850, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=22788, b4=127:11:12:13:14:15:16:128 } - { ipv4=178.79.150.233, psid=54192, b4=127:11:12:13:14:15:16:128 } -} diff --git a/src/program/lwaftr/tests/data/vlan/binding-table.txt b/src/program/lwaftr/tests/data/vlan/binding-table.txt deleted file mode 120000 index 3ceaa46b21..0000000000 --- a/src/program/lwaftr/tests/data/vlan/binding-table.txt +++ /dev/null @@ -1 +0,0 @@ -../binding-table.txt \ No newline at end of file diff --git a/src/program/snabbvmx/tests/conf/binding_table.txt b/src/program/snabbvmx/tests/conf/binding_table.txt deleted file mode 100644 index cdc0d079e6..0000000000 --- a/src/program/snabbvmx/tests/conf/binding_table.txt +++ /dev/null @@ -1,199 +0,0 @@ -psid_map { - 193.5.1.100 { psid_length=6, shift=10 } - 193.5.1.101 { psid_length=6, shift=10 } - 193.5.1.102 { psid_length=6, shift=10 } -} -br_addresses { - fc00::100 -} -softwires { - { ipv4=193.5.1.100, psid=1, b4=fc00:1:2:3:4:5:0:7e } - { ipv4=193.5.1.100, psid=2, b4=fc00:1:2:3:4:5:0:7f } - { ipv4=193.5.1.100, psid=3, b4=fc00:1:2:3:4:5:0:80 } - { ipv4=193.5.1.100, psid=4, b4=fc00:1:2:3:4:5:0:81 } - { ipv4=193.5.1.100, psid=5, b4=fc00:1:2:3:4:5:0:82 } - { ipv4=193.5.1.100, psid=6, b4=fc00:1:2:3:4:5:0:83 } - { ipv4=193.5.1.100, psid=7, b4=fc00:1:2:3:4:5:0:84 } - { ipv4=193.5.1.100, psid=8, b4=fc00:1:2:3:4:5:0:85 } - { ipv4=193.5.1.100, psid=9, b4=fc00:1:2:3:4:5:0:86 } - { ipv4=193.5.1.100, psid=10, b4=fc00:1:2:3:4:5:0:87 } - { ipv4=193.5.1.100, psid=11, b4=fc00:1:2:3:4:5:0:88 } - { ipv4=193.5.1.100, psid=12, b4=fc00:1:2:3:4:5:0:89 } - { ipv4=193.5.1.100, psid=13, b4=fc00:1:2:3:4:5:0:8a } - { ipv4=193.5.1.100, psid=14, b4=fc00:1:2:3:4:5:0:8b } - { ipv4=193.5.1.100, psid=15, b4=fc00:1:2:3:4:5:0:8c } - { ipv4=193.5.1.100, psid=16, b4=fc00:1:2:3:4:5:0:8d } - { ipv4=193.5.1.100, psid=17, b4=fc00:1:2:3:4:5:0:8e } - { ipv4=193.5.1.100, psid=18, b4=fc00:1:2:3:4:5:0:8f } - { ipv4=193.5.1.100, psid=19, b4=fc00:1:2:3:4:5:0:90 } - { ipv4=193.5.1.100, psid=20, b4=fc00:1:2:3:4:5:0:91 } - { ipv4=193.5.1.100, psid=21, b4=fc00:1:2:3:4:5:0:92 } - { ipv4=193.5.1.100, psid=22, b4=fc00:1:2:3:4:5:0:93 } - { ipv4=193.5.1.100, psid=23, b4=fc00:1:2:3:4:5:0:94 } - { ipv4=193.5.1.100, psid=24, b4=fc00:1:2:3:4:5:0:95 } - { ipv4=193.5.1.100, psid=25, b4=fc00:1:2:3:4:5:0:96 } - { ipv4=193.5.1.100, psid=26, b4=fc00:1:2:3:4:5:0:97 } - { ipv4=193.5.1.100, psid=27, b4=fc00:1:2:3:4:5:0:98 } - { ipv4=193.5.1.100, psid=28, b4=fc00:1:2:3:4:5:0:99 } - { ipv4=193.5.1.100, psid=29, b4=fc00:1:2:3:4:5:0:9a } - { ipv4=193.5.1.100, psid=30, b4=fc00:1:2:3:4:5:0:9b } - { ipv4=193.5.1.100, psid=31, b4=fc00:1:2:3:4:5:0:9c } - { ipv4=193.5.1.100, psid=32, b4=fc00:1:2:3:4:5:0:9d } - { ipv4=193.5.1.100, psid=33, b4=fc00:1:2:3:4:5:0:9e } - { ipv4=193.5.1.100, psid=34, b4=fc00:1:2:3:4:5:0:9f } - { ipv4=193.5.1.100, psid=35, b4=fc00:1:2:3:4:5:0:a0 } - { ipv4=193.5.1.100, psid=36, b4=fc00:1:2:3:4:5:0:a1 } - { ipv4=193.5.1.100, psid=37, b4=fc00:1:2:3:4:5:0:a2 } - { ipv4=193.5.1.100, psid=38, b4=fc00:1:2:3:4:5:0:a3 } - { ipv4=193.5.1.100, psid=39, b4=fc00:1:2:3:4:5:0:a4 } - { ipv4=193.5.1.100, psid=40, b4=fc00:1:2:3:4:5:0:a5 } - { ipv4=193.5.1.100, psid=41, b4=fc00:1:2:3:4:5:0:a6 } - { ipv4=193.5.1.100, psid=42, b4=fc00:1:2:3:4:5:0:a7 } - { ipv4=193.5.1.100, psid=43, b4=fc00:1:2:3:4:5:0:a8 } - { ipv4=193.5.1.100, psid=44, b4=fc00:1:2:3:4:5:0:a9 } - { ipv4=193.5.1.100, psid=45, b4=fc00:1:2:3:4:5:0:aa } - { ipv4=193.5.1.100, psid=46, b4=fc00:1:2:3:4:5:0:ab } - { ipv4=193.5.1.100, psid=47, b4=fc00:1:2:3:4:5:0:ac } - { ipv4=193.5.1.100, psid=48, b4=fc00:1:2:3:4:5:0:ad } - { ipv4=193.5.1.100, psid=49, b4=fc00:1:2:3:4:5:0:ae } - { ipv4=193.5.1.100, psid=50, b4=fc00:1:2:3:4:5:0:af } - { ipv4=193.5.1.100, psid=51, b4=fc00:1:2:3:4:5:0:b0 } - { ipv4=193.5.1.100, psid=52, b4=fc00:1:2:3:4:5:0:b1 } - { ipv4=193.5.1.100, psid=53, b4=fc00:1:2:3:4:5:0:b2 } - { ipv4=193.5.1.100, psid=54, b4=fc00:1:2:3:4:5:0:b3 } - { ipv4=193.5.1.100, psid=55, b4=fc00:1:2:3:4:5:0:b4 } - { ipv4=193.5.1.100, psid=56, b4=fc00:1:2:3:4:5:0:b5 } - { ipv4=193.5.1.100, psid=57, b4=fc00:1:2:3:4:5:0:b6 } - { ipv4=193.5.1.100, psid=58, b4=fc00:1:2:3:4:5:0:b7 } - { ipv4=193.5.1.100, psid=59, b4=fc00:1:2:3:4:5:0:b8 } - { ipv4=193.5.1.100, psid=60, b4=fc00:1:2:3:4:5:0:b9 } - { ipv4=193.5.1.100, psid=61, b4=fc00:1:2:3:4:5:0:ba } - { ipv4=193.5.1.100, psid=62, b4=fc00:1:2:3:4:5:0:bb } - { ipv4=193.5.1.100, psid=63, b4=fc00:1:2:3:4:5:0:bc } - { ipv4=193.5.1.101, psid=1, b4=fc00:1:2:3:4:5:0:bd } - { ipv4=193.5.1.101, psid=2, b4=fc00:1:2:3:4:5:0:be } - { ipv4=193.5.1.101, psid=3, b4=fc00:1:2:3:4:5:0:bf } - { ipv4=193.5.1.101, psid=4, b4=fc00:1:2:3:4:5:0:c0 } - { ipv4=193.5.1.101, psid=5, b4=fc00:1:2:3:4:5:0:c1 } - { ipv4=193.5.1.101, psid=6, b4=fc00:1:2:3:4:5:0:c2 } - { ipv4=193.5.1.101, psid=7, b4=fc00:1:2:3:4:5:0:c3 } - { ipv4=193.5.1.101, psid=8, b4=fc00:1:2:3:4:5:0:c4 } - { ipv4=193.5.1.101, psid=9, b4=fc00:1:2:3:4:5:0:c5 } - { ipv4=193.5.1.101, psid=10, b4=fc00:1:2:3:4:5:0:c6 } - { ipv4=193.5.1.101, psid=11, b4=fc00:1:2:3:4:5:0:c7 } - { ipv4=193.5.1.101, psid=12, b4=fc00:1:2:3:4:5:0:c8 } - { ipv4=193.5.1.101, psid=13, b4=fc00:1:2:3:4:5:0:c9 } - { ipv4=193.5.1.101, psid=14, b4=fc00:1:2:3:4:5:0:ca } - { ipv4=193.5.1.101, psid=15, b4=fc00:1:2:3:4:5:0:cb } - { ipv4=193.5.1.101, psid=16, b4=fc00:1:2:3:4:5:0:cc } - { ipv4=193.5.1.101, psid=17, b4=fc00:1:2:3:4:5:0:cd } - { ipv4=193.5.1.101, psid=18, b4=fc00:1:2:3:4:5:0:ce } - { ipv4=193.5.1.101, psid=19, b4=fc00:1:2:3:4:5:0:cf } - { ipv4=193.5.1.101, psid=20, b4=fc00:1:2:3:4:5:0:d0 } - { ipv4=193.5.1.101, psid=21, b4=fc00:1:2:3:4:5:0:d1 } - { ipv4=193.5.1.101, psid=22, b4=fc00:1:2:3:4:5:0:d2 } - { ipv4=193.5.1.101, psid=23, b4=fc00:1:2:3:4:5:0:d3 } - { ipv4=193.5.1.101, psid=24, b4=fc00:1:2:3:4:5:0:d4 } - { ipv4=193.5.1.101, psid=25, b4=fc00:1:2:3:4:5:0:d5 } - { ipv4=193.5.1.101, psid=26, b4=fc00:1:2:3:4:5:0:d6 } - { ipv4=193.5.1.101, psid=27, b4=fc00:1:2:3:4:5:0:d7 } - { ipv4=193.5.1.101, psid=28, b4=fc00:1:2:3:4:5:0:d8 } - { ipv4=193.5.1.101, psid=29, b4=fc00:1:2:3:4:5:0:d9 } - { ipv4=193.5.1.101, psid=30, b4=fc00:1:2:3:4:5:0:da } - { ipv4=193.5.1.101, psid=31, b4=fc00:1:2:3:4:5:0:db } - { ipv4=193.5.1.101, psid=32, b4=fc00:1:2:3:4:5:0:dc } - { ipv4=193.5.1.101, psid=33, b4=fc00:1:2:3:4:5:0:dd } - { ipv4=193.5.1.101, psid=34, b4=fc00:1:2:3:4:5:0:de } - { ipv4=193.5.1.101, psid=35, b4=fc00:1:2:3:4:5:0:df } - { ipv4=193.5.1.101, psid=36, b4=fc00:1:2:3:4:5:0:e0 } - { ipv4=193.5.1.101, psid=37, b4=fc00:1:2:3:4:5:0:e1 } - { ipv4=193.5.1.101, psid=38, b4=fc00:1:2:3:4:5:0:e2 } - { ipv4=193.5.1.101, psid=39, b4=fc00:1:2:3:4:5:0:e3 } - { ipv4=193.5.1.101, psid=40, b4=fc00:1:2:3:4:5:0:e4 } - { ipv4=193.5.1.101, psid=41, b4=fc00:1:2:3:4:5:0:e5 } - { ipv4=193.5.1.101, psid=42, b4=fc00:1:2:3:4:5:0:e6 } - { ipv4=193.5.1.101, psid=43, b4=fc00:1:2:3:4:5:0:e7 } - { ipv4=193.5.1.101, psid=44, b4=fc00:1:2:3:4:5:0:e8 } - { ipv4=193.5.1.101, psid=45, b4=fc00:1:2:3:4:5:0:e9 } - { ipv4=193.5.1.101, psid=46, b4=fc00:1:2:3:4:5:0:ea } - { ipv4=193.5.1.101, psid=47, b4=fc00:1:2:3:4:5:0:eb } - { ipv4=193.5.1.101, psid=48, b4=fc00:1:2:3:4:5:0:ec } - { ipv4=193.5.1.101, psid=49, b4=fc00:1:2:3:4:5:0:ed } - { ipv4=193.5.1.101, psid=50, b4=fc00:1:2:3:4:5:0:ee } - { ipv4=193.5.1.101, psid=51, b4=fc00:1:2:3:4:5:0:ef } - { ipv4=193.5.1.101, psid=52, b4=fc00:1:2:3:4:5:0:f0 } - { ipv4=193.5.1.101, psid=53, b4=fc00:1:2:3:4:5:0:f1 } - { ipv4=193.5.1.101, psid=54, b4=fc00:1:2:3:4:5:0:f2 } - { ipv4=193.5.1.101, psid=55, b4=fc00:1:2:3:4:5:0:f3 } - { ipv4=193.5.1.101, psid=56, b4=fc00:1:2:3:4:5:0:f4 } - { ipv4=193.5.1.101, psid=57, b4=fc00:1:2:3:4:5:0:f5 } - { ipv4=193.5.1.101, psid=58, b4=fc00:1:2:3:4:5:0:f6 } - { ipv4=193.5.1.101, psid=59, b4=fc00:1:2:3:4:5:0:f7 } - { ipv4=193.5.1.101, psid=60, b4=fc00:1:2:3:4:5:0:f8 } - { ipv4=193.5.1.101, psid=61, b4=fc00:1:2:3:4:5:0:f9 } - { ipv4=193.5.1.101, psid=62, b4=fc00:1:2:3:4:5:0:fa } - { ipv4=193.5.1.101, psid=63, b4=fc00:1:2:3:4:5:0:fb } - { ipv4=193.5.1.102, psid=1, b4=fc00:1:2:3:4:5:0:fc } - { ipv4=193.5.1.102, psid=2, b4=fc00:1:2:3:4:5:0:fd } - { ipv4=193.5.1.102, psid=3, b4=fc00:1:2:3:4:5:0:fe } - { ipv4=193.5.1.102, psid=4, b4=fc00:1:2:3:4:5:0:ff } - { ipv4=193.5.1.102, psid=5, b4=fc00:1:2:3:4:5:0:100 } - { ipv4=193.5.1.102, psid=6, b4=fc00:1:2:3:4:5:0:101 } - { ipv4=193.5.1.102, psid=7, b4=fc00:1:2:3:4:5:0:102 } - { ipv4=193.5.1.102, psid=8, b4=fc00:1:2:3:4:5:0:103 } - { ipv4=193.5.1.102, psid=9, b4=fc00:1:2:3:4:5:0:104 } - { ipv4=193.5.1.102, psid=10, b4=fc00:1:2:3:4:5:0:105 } - { ipv4=193.5.1.102, psid=11, b4=fc00:1:2:3:4:5:0:106 } - { ipv4=193.5.1.102, psid=12, b4=fc00:1:2:3:4:5:0:107 } - { ipv4=193.5.1.102, psid=13, b4=fc00:1:2:3:4:5:0:108 } - { ipv4=193.5.1.102, psid=14, b4=fc00:1:2:3:4:5:0:109 } - { ipv4=193.5.1.102, psid=15, b4=fc00:1:2:3:4:5:0:10a } - { ipv4=193.5.1.102, psid=16, b4=fc00:1:2:3:4:5:0:10b } - { ipv4=193.5.1.102, psid=17, b4=fc00:1:2:3:4:5:0:10c } - { ipv4=193.5.1.102, psid=18, b4=fc00:1:2:3:4:5:0:10d } - { ipv4=193.5.1.102, psid=19, b4=fc00:1:2:3:4:5:0:10e } - { ipv4=193.5.1.102, psid=20, b4=fc00:1:2:3:4:5:0:10f } - { ipv4=193.5.1.102, psid=21, b4=fc00:1:2:3:4:5:0:110 } - { ipv4=193.5.1.102, psid=22, b4=fc00:1:2:3:4:5:0:111 } - { ipv4=193.5.1.102, psid=23, b4=fc00:1:2:3:4:5:0:112 } - { ipv4=193.5.1.102, psid=24, b4=fc00:1:2:3:4:5:0:113 } - { ipv4=193.5.1.102, psid=25, b4=fc00:1:2:3:4:5:0:114 } - { ipv4=193.5.1.102, psid=26, b4=fc00:1:2:3:4:5:0:115 } - { ipv4=193.5.1.102, psid=27, b4=fc00:1:2:3:4:5:0:116 } - { ipv4=193.5.1.102, psid=28, b4=fc00:1:2:3:4:5:0:117 } - { ipv4=193.5.1.102, psid=29, b4=fc00:1:2:3:4:5:0:118 } - { ipv4=193.5.1.102, psid=30, b4=fc00:1:2:3:4:5:0:119 } - { ipv4=193.5.1.102, psid=31, b4=fc00:1:2:3:4:5:0:11a } - { ipv4=193.5.1.102, psid=32, b4=fc00:1:2:3:4:5:0:11b } - { ipv4=193.5.1.102, psid=33, b4=fc00:1:2:3:4:5:0:11c } - { ipv4=193.5.1.102, psid=34, b4=fc00:1:2:3:4:5:0:11d } - { ipv4=193.5.1.102, psid=35, b4=fc00:1:2:3:4:5:0:11e } - { ipv4=193.5.1.102, psid=36, b4=fc00:1:2:3:4:5:0:11f } - { ipv4=193.5.1.102, psid=37, b4=fc00:1:2:3:4:5:0:120 } - { ipv4=193.5.1.102, psid=38, b4=fc00:1:2:3:4:5:0:121 } - { ipv4=193.5.1.102, psid=39, b4=fc00:1:2:3:4:5:0:122 } - { ipv4=193.5.1.102, psid=40, b4=fc00:1:2:3:4:5:0:123 } - { ipv4=193.5.1.102, psid=41, b4=fc00:1:2:3:4:5:0:124 } - { ipv4=193.5.1.102, psid=42, b4=fc00:1:2:3:4:5:0:125 } - { ipv4=193.5.1.102, psid=43, b4=fc00:1:2:3:4:5:0:126 } - { ipv4=193.5.1.102, psid=44, b4=fc00:1:2:3:4:5:0:127 } - { ipv4=193.5.1.102, psid=45, b4=fc00:1:2:3:4:5:0:128 } - { ipv4=193.5.1.102, psid=46, b4=fc00:1:2:3:4:5:0:129 } - { ipv4=193.5.1.102, psid=47, b4=fc00:1:2:3:4:5:0:12a } - { ipv4=193.5.1.102, psid=48, b4=fc00:1:2:3:4:5:0:12b } - { ipv4=193.5.1.102, psid=49, b4=fc00:1:2:3:4:5:0:12c } - { ipv4=193.5.1.102, psid=50, b4=fc00:1:2:3:4:5:0:12d } - { ipv4=193.5.1.102, psid=51, b4=fc00:1:2:3:4:5:0:12e } - { ipv4=193.5.1.102, psid=52, b4=fc00:1:2:3:4:5:0:12f } - { ipv4=193.5.1.102, psid=53, b4=fc00:1:2:3:4:5:0:130 } - { ipv4=193.5.1.102, psid=54, b4=fc00:1:2:3:4:5:0:131 } - { ipv4=193.5.1.102, psid=55, b4=fc00:1:2:3:4:5:0:132 } - { ipv4=193.5.1.102, psid=56, b4=fc00:1:2:3:4:5:0:133 } - { ipv4=193.5.1.102, psid=57, b4=fc00:1:2:3:4:5:0:134 } - { ipv4=193.5.1.102, psid=58, b4=fc00:1:2:3:4:5:0:135 } - { ipv4=193.5.1.102, psid=59, b4=fc00:1:2:3:4:5:0:136 } - { ipv4=193.5.1.102, psid=60, b4=fc00:1:2:3:4:5:0:137 } - { ipv4=193.5.1.102, psid=61, b4=fc00:1:2:3:4:5:0:138 } - { ipv4=193.5.1.102, psid=62, b4=fc00:1:2:3:4:5:0:139 } - { ipv4=193.5.1.102, psid=63, b4=fc00:1:2:3:4:5:0:13a } -} From 3078cb7147818ab2c963ebefffe506121b965a30 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 15:50:50 +0100 Subject: [PATCH 178/631] More legacy lwaftr code migration --- src/apps/lwaftr/stream.lua | 163 ------------------ .../migrate_configuration}/conf_parser.lua | 0 .../migrate_configuration.lua | 2 +- 3 files changed, 1 insertion(+), 164 deletions(-) delete mode 100644 src/apps/lwaftr/stream.lua rename src/{apps/lwaftr => program/lwaftr/migrate_configuration}/conf_parser.lua (100%) diff --git a/src/apps/lwaftr/stream.lua b/src/apps/lwaftr/stream.lua deleted file mode 100644 index 1a730ac800..0000000000 --- a/src/apps/lwaftr/stream.lua +++ /dev/null @@ -1,163 +0,0 @@ -module(..., package.seeall) - -local ffi = require("ffi") -local S = require("syscall") -local lib = require("core.lib") - -local function round_up(x, y) return y*math.ceil(x/y) end - -function open_output_byte_stream(filename) - local fd, err = - S.open(filename, "creat, wronly, trunc", "rusr, wusr, rgrp, roth") - if not fd then - error("error opening output file "..filename..": "..tostring(err)) - end - local ret = { written = 0, name = filename } - function ret:close() - fd:close() - end - function ret:error(msg) - self:close() - error('while writing file '..filename..': '..msg) - end - function ret:write(ptr, size) - assert(size) - ptr = ffi.cast("uint8_t*", ptr) - local to_write = size - while to_write > 0 do - local written, err = S.write(fd, ptr, to_write) - if not written then self:error(err) end - ptr = ptr + written - self.written = self.written + written - to_write = to_write - written - end - end - function ret:write_ptr(ptr) - self:align(ffi.alignof(ptr)) - self:write(ptr, ffi.sizeof(ptr)) - end - function ret:write_array(ptr, type, count) - self:align(ffi.alignof(type)) - self:write(ptr, ffi.sizeof(type) * count) - end - function ret:align(alignment) - local padding = round_up(self.written, alignment) - self.written - self:write(string.rep(' ', padding), padding) - end - return ret -end - -local function mktemp(name, mode) - if not mode then mode = "rusr, wusr, rgrp, roth" end - -- FIXME: If nothing seeds math.random, this produces completely - -- predictable numbers. - local t = math.random(1e7) - local tmpnam, fd, err - for i = t, t+10 do - tmpnam = name .. '.' .. i - fd, err = S.open(tmpnam, "creat, wronly, excl", mode) - if fd then - fd:close() - return tmpnam, nil - end - i = i + 1 - end - return nil, err -end - -function open_temporary_output_byte_stream(target) - local tmp_file, err = mktemp(target) - if not tmp_file then - local dir = lib.dirname(target) - error("failed to create temporary file in "..dir..": "..tostring(err)) - end - local stream = open_output_byte_stream(tmp_file) - function stream:close_and_rename() - self:close() - local res, err = S.rename(tmp_file, target) - if not res then - error("failed to rename "..tmp_file.." to "..target..": "..err) - end - end - return stream -end - --- FIXME: Try to copy file into huge pages? -function open_input_byte_stream(filename) - local fd, err = S.open(filename, "rdonly") - if not fd then return - error("error opening "..filename..": "..tostring(err)) - end - local stat = S.fstat(fd) - local size = stat.size - local mem, err = S.mmap(nil, size, 'read, write', 'private', fd, 0) - fd:close() - if not mem then error("mmap failed: " .. tostring(err)) end - mem = ffi.cast("uint8_t*", mem) - local pos = 0 - local ret = { - name=filename, - mtime_sec=stat.st_mtime, - mtime_nsec=stat.st_mtime_nsec - } - function ret:close() - -- FIXME: Currently we don't unmap any memory. - -- S.munmap(mem, size) - mem, pos = nil, nil - end - function ret:error(msg) - error('while reading file '..filename..': '..msg) - end - function ret:read(count) - assert(count >= 0) - local ptr = mem + pos - pos = pos + count - if pos > size then - self:error('unexpected EOF') - end - return ptr - end - function ret:align(alignment) - self:read(round_up(pos, alignment) - pos) - end - function ret:seek(new_pos) - assert(new_pos >= 0) - assert(new_pos <= size) - pos = new_pos - end - function ret:read_ptr(type) - ret:align(ffi.alignof(type)) - return ffi.cast(ffi.typeof('$*', type), ret:read(ffi.sizeof(type))) - end - function ret:read_array(type, count) - ret:align(ffi.alignof(type)) - return ffi.cast(ffi.typeof('$*', type), - ret:read(ffi.sizeof(type) * count)) - end - function ret:read_char() - return ffi.string(ret:read(1), 1) - end - function ret:as_text_stream(len) - local end_pos = size - if len then end_pos = pos + len end - return { - name = ret.name, - mtime_sec = ret.mtime_sec, - mtime_nsec = ret.mtime_nsec, - read = function(self, n) - assert(n==1) - if pos == end_pos then return nil end - return ret:read_char() - end, - close = function() ret:close() end - } - end - return ret -end - --- You're often better off using Lua's built-in files. This is here --- because it gives a file-like object whose FD you can query, for --- example to get its mtime. -function open_input_text_stream(filename) - return open_input_byte_stream(filename):as_text_stream() -end diff --git a/src/apps/lwaftr/conf_parser.lua b/src/program/lwaftr/migrate_configuration/conf_parser.lua similarity index 100% rename from src/apps/lwaftr/conf_parser.lua rename to src/program/lwaftr/migrate_configuration/conf_parser.lua diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 8b979bee15..2bcdc8af92 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -11,8 +11,8 @@ local cltable = require('lib.cltable') local util = require('lib.yang.util') local yang = require('lib.yang.yang') local stream = require('lib.yang.stream') -local Parser = require("apps.lwaftr.conf_parser").Parser local binding_table = require("apps.lwaftr.binding_table") +local Parser = require("program.lwaftr.migrate_configuration.conf_parser").Parser local br_address_t = ffi.typeof('uint8_t[16]') local SOFTWIRE_TABLE_LOAD_FACTOR = 0.4 From 952c6a43a722c357a40bed26328739638f7b7d44 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 15:56:25 +0100 Subject: [PATCH 179/631] Fix snabb lwaftr bench --- src/program/lwaftr/bench/bench.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 3d3cf5a098..fb16d9c800 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -29,7 +29,7 @@ end function run(args) local opts, conf_file, inv4_pcap, inv6_pcap = parse_args(args) - local conf = require('apps.lwaftr.conf').load_config(conf_file) + local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local c = config.new() setup.load_bench(c, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') From 5c5f64ad237e53dd0a278c43a8dd82942a4ba6d0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 17:20:09 +0100 Subject: [PATCH 180/631] Update lwaftr docs * src/program/lwaftr/doc/README.configuration.md: Update. * src/program/lwaftr/doc/README.bindingtable.md: Remove. --- src/program/lwaftr/doc/README.bindingtable.md | 116 ------ .../lwaftr/doc/README.configuration.md | 378 +++++++++--------- 2 files changed, 190 insertions(+), 304 deletions(-) delete mode 100644 src/program/lwaftr/doc/README.bindingtable.md diff --git a/src/program/lwaftr/doc/README.bindingtable.md b/src/program/lwaftr/doc/README.bindingtable.md deleted file mode 100644 index d3e10174ab..0000000000 --- a/src/program/lwaftr/doc/README.bindingtable.md +++ /dev/null @@ -1,116 +0,0 @@ -# Binding tables - -A binding table is a collection of softwires (tunnels). One endpoint -of the softwire is in the AFTR and the other is in the B4. A -softwire provisions an IPv4 address (or a part of an IPv4 address) to -a customer behind a B4. The B4 arranges for all IPv4 traffic to be -encapsulated in IPv6 and sent to the AFTR; the AFTR does the reverse. -The binding table is how the AFTR knows which B4 is associated with -an incoming packet. - -## File structure - -There are three parts of a binding table: the PSID info map, the -border router (BR) address table, and the softwire map. Grammatically -they appear in the file in the following order,: - -``` - psid_map { - ... - } - br_addresses { - ... - } - softwires { - ... - } -``` - -## PSID info map - -The PSID info map defines the set of IPv4 addresses that are provisioned -by an lwAFTR. It also defines the way in which those addresses are -shared, by specifying the "psid_length" and "shift" parameters for each -address. See RFC 7597 for more details on the PSID scheme for how to -share IPv4 addresses. The `psid_map` clause is composed of a list of -entries, each of which specifying a set of IPv4 address and the PSID -parameters. In this and other clauses, newlines and other white space -are insignificant. For example: - -`` - psid_map { - 1.2.3.4 { psid_length=10 } - 5.6.7.8 { psid_length=5, shift=11 } - ... - } -`` - -An entry's `psid_length` and `shift` parameters must necessarily add up -to 16, so it is sufficient to specify just one of them. If neither are -specified, they default to 0 and 16, respectively. - -The addresses may be specified as ranges or lists of ranges as well: - -``` - psid_map { - 1.2.3.4 { psid_length=10 } - 2.0.0.0, 3.0.0.0, 4.0.0.0-4.1.2.3 { psid_length=7 } - } -``` - -The set of IPv4 address ranges specified in the PSID info map must be -disjoint. - -## Border router addresses - -Next, the `br_addresses` clause lists the set of IPv6 addresses to -associate with the lwAFTR. These are the "border router" addresses. -For a usual deployment there will be one main address and possibly some -additional ones. For example: - -``` - br_addresses { - 8:9:a:b:c:d:e:f, - 1E:1:1:1:1:1:1:af, - 1E:2:2:2:2:2:2:af - } -``` - -## Softwires - -Finally, the `softwires` clause defines the set of softwires to -provision. Each softwire associates an IPv4 address, a PSID, and a B4 -address. For example: - -``` - softwires { - { ipv4=178.79.150.233, psid=80, b4=127:2:3:4:5:6:7:128 } - { ipv4=178.79.150.233, psid=2300, b4=127:11:12:13:14:15:16:128 } - ... - } -``` - -By default, a softwire is associated with the first entry in -`br_addresses` (`aftr=0`). To associate the tunnel with a different -border router, specify it by index: - -``` - softwires { - { ipv4=178.79.150.233, psid=80, b4=127:2:3:4:5:6:7:128, aftr=0 } - { ipv4=178.79.150.233, psid=2300, b4=127:11:12:13:14:15:16:128, aftr=42 } - ... - } -``` - -## Compiling binding tables - -Internally, the lwAFTR uses the binding table in a compiled format. -When a lwAFTR is started, it will automatically compile its binding -table if needed. However for large tables (millions of entries) this -can take a second or two, so it can still be useful to compile the -binding table ahead of time. - -Use the `snabb lwaftr compile-binding-table` command to compile a -binding table ahead of time. If you do this, you can use the -`snabb lwaftr control PID reload` command to tell the Snabb process -with the given *PID* to reload the table. diff --git a/src/program/lwaftr/doc/README.configuration.md b/src/program/lwaftr/doc/README.configuration.md index feb351a0d2..82e84230a5 100644 --- a/src/program/lwaftr/doc/README.configuration.md +++ b/src/program/lwaftr/doc/README.configuration.md @@ -1,193 +1,208 @@ # Configuration -The lwAFTR is configured by a text file. Where applicable, default values can -be found in [the code](../../../apps/lwaftr/conf.lua#L72). +The lwAFTR's configuration is modelled by a +[YANG](https://tools.ietf.org/html/rfc6020) schema, +[snabb-softwire-v1](../../../lib/yang/snabb-softwire-v1.yang). + +The lwAFTR takes its configuration from the user in the form of a text +file. That file's grammar is derived from the YANG schema (see the +[Snabb YANG README](../../../lib/yang/README.md). Here's an example: + +``` +// IPv4 interface. +external-interface { + allow-incoming-icmp true; + // Limit ICMP error transmit rate to PACKETS/PERIOD. + error-rate-limiting { + packets 600000; + period 2; + } + // Generate ICMP errors at all? + generate-icmp-errors true; + // Basic parameters. + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + // vlan-tag 42; + // Where to go next. Either one will suffice; if you specify the IP, + // the next-hop MAC will be determined by ARP. + next-hop { + mac 68:68:68:68:68:68; + ip 1.2.3.4; + } + // Control the size of the fragment reassembly buffer. + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +// The same thing as for the external interface, but on the IPv6 side +// and with IPv6 addresses. +internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + // One more interesting thing -- here we control whether to support + // routing traffic between two B4s. + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + // vlan-tag 64; + next-hop { + mac 44:44:44:44:44:44; + // NDP instead of ARP of course. + ip 7:8:9:a:b:c:d:e; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } +} +// Now the binding table! 3 parts: PSID map, BR address table, and +// softwire set. See description below for details. +binding-table { + psid-map { addr 178.79.150.15; psid-length 4; } + psid-map { addr 178.79.150.233; psid-length 16; } + psid-map { addr 178.79.150.2; psid-length 16; } + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } +} +``` + +Basically there's an `external-interface` section defining the +parameters around the IPv4 interface that communicates with the +internet, an `internal-interface` section doing the same for the IPv6 +side that communicates with the B4s, and then the `binding-table` that +declares the set of softwires. + +## Compiling conigurations + +When a lwAFTR is started, it will automatically compile its +configuration if it does not find a compiled configuration that's fresh. +However for large configurations with millions of binding table entries, +this can take a second or two, so it can still be useful to compile the +configuration ahead of time. + +Use the `snabb lwaftr compile-configuration` command to compile a +configuration ahead of time. If you do this, you can use the `snabb +lwaftr control PID reload` command to tell the Snabb process with the +given *PID* to reload the table. + +## In-depth configuration explanation + +See the embedded descriptions in the +[snabb-softwire-v1](../../../lib/yang/snabb-softwire-v1.yang) schema +file. + +## Binding tables + +A binding table is a collection of softwires (tunnels). One endpoint +of the softwire is in the AFTR and the other is in the B4. A +softwire provisions an IPv4 address (or a part of an IPv4 address) to +a customer behind a B4. The B4 arranges for all IPv4 traffic to be +encapsulated in IPv6 and sent to the AFTR; the AFTR does the reverse. +The binding table is how the AFTR knows which B4 is associated with +an incoming packet. + +In the Snabb lwAFTR there are three parts of a binding table: the PSID +info map, the border router (BR) address table, and the softwire map. + +### PSID info map + +The PSID info map defines the set of IPv4 addresses that are provisioned +by an lwAFTR. It also defines the way in which those addresses are +shared, by specifying the "psid-length" and "shift" parameters for each +address. See RFC 7597 for more details on the PSID scheme for how to +share IPv4 addresses. IPv4 addresses are added to the `psid-map` with +statements like this: + +`` + psid-map { + addr 178.79.150.3; + end-addr 178.100.150.3; + psid-length 6; + reserved-ports-bit-count 0; + shift 10; + } +`` + +`end-addr`, `reserved-ports-bit-count`, and `shift` are all optional. + +An entry's `psid-length`, `shift` and `reserved-ports-bit-count` +parameters must necessarily add up to 16, so it is sufficient to specify +just two of them. Actually it's sufficient to just specify the +`psid-length`, which is the only required one. +`reserved-ports-bit-count` defaults to 0, and `shift` defaults to `16 - +psid-length`. + +### Border router addresses + +Next, the `br-address` clauses define the set of IPv6 addresses to +associate with the lwAFTR. These are the "border router" addresses. +For a usual deployment there will be one main address and possibly some +additional ones. For example: -Here's an example: - -``` -aftr_ipv4_ip = 10.10.10.10 -aftr_ipv6_ip = 8:9:a:b:c:d:e:f -aftr_mac_b4_side = 22:22:22:22:22:22 -aftr_mac_inet_side = 12:12:12:12:12:12 -next_hop6_mac = 44:44:44:44:44:44 -# next_hop_ipv6_addr = fd00::1 -inet_mac = 52:54:00:00:00:01 -# next_hop_ipv4_addr = 192.168.0.1 -binding_table = path/to/binding-table.txt -hairpinning = true -icmpv6_rate_limiter_n_packets=3e5 -icmpv6_rate_limiter_n_seconds=4 -inet_mac = 68:68:68:68:68:68 -ipv4_mtu = 1460 -ipv6_mtu = 1500 -max_fragments_per_reassembly_packet = 1, -max_ipv6_reassembly_packets = 10, -max_ipv4_reassembly_packets = 10, -policy_icmpv4_incoming = ALLOW -policy_icmpv6_incoming = ALLOW -policy_icmpv4_outgoing = ALLOW -policy_icmpv6_outgoing = ALLOW -v4_vlan_tag = 1234 -v6_vlan_tag = 42 -vlan_tagging = true -# ipv4_ingress_filter = "ip" -# ipv4_egress_filter = "ip" -# ipv6_ingress_filter = "ip6" -# ipv6_egress_filter = "ip6" -``` - -The lwAFTR is associated with two physical network cards. One of these cards -faces the internet; traffic over it is IPv4. The other faces the IPv6-only -internal network, and communicates primarily with B4s. - -## Line-by-line explanation - -### L2 and L3 addresses of the lwAFTR - -First, we set the IP and MAC addresses for both interfaces: - -``` -aftr_ipv4_ip = 10.10.10.10 -aftr_ipv6_ip = 8:9:a:b:c:d:e:f -aftr_mac_b4_side = 22:22:22:22:22:22 -aftr_mac_inet_side = 12:12:12:12:12:12 -``` - -This associates **12:12:12:12:12:12** and **10.10.10.10** with the -internet-facing NIC, and **8:9:a:b:c:d:e:f** and **22:22:22:22:22:22** with the -NIC facing the internal network. - -### L2 next hops - -Normally you might expect to just set a default IPv4 and IPv6 gateway -and have the lwAFTR figure out the next hop ethernet addresses on its -own. However the lwAFTR doesn't support -[ARP](https://en.wikipedia.org/wiki/Address_Resolution_Protocol) yet. - -The lwAFTR assumes that it will talk directly to only one host on each -side, and provides these configuration options to specify the L2 -addresses of those hosts. - -``` -next_hop6_mac = 44:44:44:44:44:44 -inet_mac = 68:68:68:68:68:68 -``` - -The lwAFTR can talk to any host, but assumes that the above ones are the -next hop. - -Alternatively, it is possible to use IP addresses for the next hops. The lwAFTR -will resolve the IP addresses to their correspondent MAC addresses, using -the NDP and ARP protocols. - -``` -next_hop_ipv6_addr = fd00::1 -next_hop_ipv4_addr = 192.168.0.1 ``` - -### The binding table - + br-address 8:9:a:b:c:d:e:f; + br-address 1E:1:1:1:1:1:1:af; + ... ``` -binding_table = path/to/binding-table.txt -``` - -See [README.bindingtable.md](README.bindingtable.md) for binding table -details. Note that you can compile the binding table beforehand; again, -see [README.bindingtable.md](README.bindingtable.md). -If the path to the binding table is a relative path, it will be relative -to the location of the configuration file. Enclose the path in single -or double quotes if the path contains spaces. +### Softwires -### Hairpinning +Finally, the `softwire` clauses define the set of softwires to +provision. Each softwire associates an IPv4 address, a PSID, and a B4 +address. For example: ``` -hairpinning = true + softwire { ipv4 178.79.150.233; psid 80; b4-ipv6 127:2:3:4:5:6:7:128; } ``` -Configurable hairpinning is a requirement of [RFC -7596](https://tools.ietf.org/html/rfc7596); it can be true or false. - -### Rate-limiting of ICMP error messages +By default, a softwire is associated with the first `br-address` +(`br 0;`). To associate the tunnel with a different border router, +specify it by index: ``` -icmpv6_rate_limiter_n_packets=3e5 -icmpv6_rate_limiter_n_seconds=4 + softwire { ipv4 178.79.150.233; psid 80; b4-ipv6 127:2:3:4:5:6:7:128; aftr 0; } ``` -ICMPv6 rate limiting is mandated by several RFCs. This example says that the -lwAFTR can send at most 300,000 (3 * 10^5) ICMPv6 packets per 4 seconds. -Lower values are recommended for non-experimental use. - -### MTU +## Ingress and egress filters +Both the `internal-interface` and `external-interface` configuration +blocks support `ingress-filter` and `egress-filter` options. ``` -ipv4_mtu = 1460 -ipv6_mtu = 1500 +... +ingress-filter "ip"; +egress-filter "ip"; ``` -The MTU settings are used to determine whether a packet needs to be -fragmented. The current MTU handling is otherwise underdeveloped. It -is not dynamically updated on receiving ICMP packet too big messages. - -### Packet reassembly - -``` -max_fragments_per_reassembly_packet = 1, -max_ipv6_reassembly_packets = 10, -max_ipv4_reassembly_packets = 10, -``` - -A packet might be split into several fragments, from which it will be -reassembled. The maximum allowed number of fragments per packet can be set. -The maximum simultaneous number of packets undergoing reassembly can also be -set separately for IPv4 and IPv6. - -### ICMP handling policies - -``` -policy_icmpv4_incoming = ALLOW -policy_icmpv6_incoming = ALLOW -policy_icmpv4_outgoing = ALLOW -policy_icmpv6_outgoing = ALLOW -``` - -The lwAFTR can be configured to `ALLOW` or `DROP` incoming and outgoing -ICMPv4 and ICMPv6 messages. If a finer granularity of control is -desired, contact the development team via github or email. - -### VLAN tagging - -``` -v4_vlan_tag = 1234 -v6_vlan_tag = 42 -vlan_tagging = tru -``` - -Enable/disable 802.1Q Ethernet tagging with 'vlan_tagging'. - -If it is enabled, set one tag per interface to tag outgoing packets with, and -assume that incoming packets are tagged. If it is 'false', v4_vlan_tag and -v6_vlan_tag are currently optional (and unused). - -Values of `v4_vlan_tag` and `v6_vlan_tag` represent the identifier value in a -VLAN tag. It must be a value between 0 and 4095. - -More sophisticated support, including for mixes of tagged/untagged packets, -will be provided upon request. - -### Ingress and egress filters - -``` -# ipv4_ingress_filter = "ip" -# ipv4_egress_filter = "ip" -# ipv6_ingress_filter = "ip6" -# ipv6_egress_filter = "ip6" -``` - -In the example configuration these entries are commented out by the `#` -character. If uncommented, the right-hand-side should be a +If given these filters should be a [pflang](https://github.com/Igalia/pflua/blob/master/doc/pflang.md) filter. Pflang is the language of `tcpdump`, `libpcap`, and other tools. @@ -199,7 +214,8 @@ the lwAFTR. It might help to think of the filter as being "whitelists" "blacklist" filter, use the `not` pflang operator: ``` -ipv4_ingress_filter = "not ip6" +// Reject IPv6. +ingress-filter "not ip6"; ``` You might need to use parentheses so that you are applying the `not` to @@ -210,7 +226,7 @@ stripped. Here is a more complicated example: ``` -ipv6_egress_filter = " +egress-filter " ip6 and not ( (icmp6 and src net 3ffe:501:0:1001::2/128 and @@ -222,22 +238,8 @@ ipv6_egress_filter = " src portrange 2397-2399 and dst port 53) ) -" -``` - -As filter definitions can be a bit unmanageable as part of the -configuration, you can also load filters from a file. To do this, start -the filter configuration like with `<` and follow it immediately with a -file name. - -``` -ipv4_ingress_filter = Date: Wed, 9 Nov 2016 17:22:02 +0100 Subject: [PATCH 181/631] More doc tweaks --- src/program/lwaftr/doc/README.breaking_changes.md | 4 ---- src/program/lwaftr/doc/README.md | 13 ++++++------- 2 files changed, 6 insertions(+), 11 deletions(-) delete mode 100644 src/program/lwaftr/doc/README.breaking_changes.md diff --git a/src/program/lwaftr/doc/README.breaking_changes.md b/src/program/lwaftr/doc/README.breaking_changes.md deleted file mode 100644 index a321220f3c..0000000000 --- a/src/program/lwaftr/doc/README.breaking_changes.md +++ /dev/null @@ -1,4 +0,0 @@ -# Breaking changes since the October 2015 alpha release - -- b4_mac has been renamed next_hop6_mac, as the next hop on the internal IPv6 - network is not actually a B4, just on the route to them. diff --git a/src/program/lwaftr/doc/README.md b/src/program/lwaftr/doc/README.md index 27eb608753..6ac47213e3 100644 --- a/src/program/lwaftr/doc/README.md +++ b/src/program/lwaftr/doc/README.md @@ -91,16 +91,15 @@ access to a NIC. ### Configuration There are a lot of configuration knobs! See the -[Configuration](./README.configuration.md) page, for general configuration, and -[Binding table](./README.bindingtable.md) page, for more on binding tables. +[Configuration](./README.configuration.md) page. ### Running the lwAFTR -You have a binding table and a configuration: great, you're finally -ready to run the lwAFTR! The only tricky part is making sure you're -using the right network interfaces. See [Running](./README.running.md), -and be sure to check [Performance](./README.performance.md) to make sure -you're getting all the lwAFTR can give. +You have a configuration: great, you're finally ready to run the lwAFTR! +The only tricky part is making sure you're using the right network +interfaces. See [Running](./README.running.md), and be sure to check +[Performance](./README.performance.md) to make sure you're getting all +the lwAFTR can give. The lwAFTR processes traffic between any NIC supported by Snabb, which mainly means Intel 82599 10 Gb adapters. It's also possible to run on From 008f5e54d854a21fab4bf0aff4a059406f5d8a71 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 17:28:18 +0100 Subject: [PATCH 182/631] Update changelog for next lwaftr release --- src/program/lwaftr/doc/CHANGELOG.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 294762c05a..e9e93bdb15 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,10 +1,28 @@ # Change Log -## Pre-release changes +## [3.0] - 2016-11-09 + +A change to migrate the lwAFTR to use a new YANG-based configuration. + + * New configuration format based on YANG. To migrate old + configurations, run "snabb lwaftr migrate-configation old.conf" on + the old configuration. See the snabb-softwire-v1.yang schema or + README.configuration.md for full details on the new configuration + format. * Send ICMPv6 unreachable messages from the most appropriate source address available (the one associated with a B4 if possible, or else the one the - packet one is in reply to had as a destination.) + packet one is in reply to had as a destination.) + + * Add support for ARP resolution of the next hop on the external + interface. + + * Add support for virtualized control planes via Snabb vMX. + + * Add many more counters, used to diagnose the path that packets take + in the lwAFTR. See README.counters.md for more. + + * Many updates from upstream Snabb. ## [2.10] - 2016-06-17 From b6dbea296cf43d95bcfb6eb9c9819cf0c52404d5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 17:29:49 +0100 Subject: [PATCH 183/631] Update README.configuration.md --- src/program/lwaftr/doc/README.configuration.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/doc/README.configuration.md b/src/program/lwaftr/doc/README.configuration.md index 82e84230a5..48f55fa088 100644 --- a/src/program/lwaftr/doc/README.configuration.md +++ b/src/program/lwaftr/doc/README.configuration.md @@ -5,8 +5,9 @@ The lwAFTR's configuration is modelled by a [snabb-softwire-v1](../../../lib/yang/snabb-softwire-v1.yang). The lwAFTR takes its configuration from the user in the form of a text -file. That file's grammar is derived from the YANG schema (see the -[Snabb YANG README](../../../lib/yang/README.md). Here's an example: +file. That file's grammar is derived from the YANG schema; see the +[Snabb YANG README](../../../lib/yang/README.md) for full details. +Here's an example: ``` // IPv4 interface. From be77b4fa236e18055d4a1e9ccae6dfc6a2e7b128 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 18:34:16 +0100 Subject: [PATCH 184/631] Fix data serialization bugs --- src/lib/yang/data.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 47af891adf..2c8edaacff 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -188,7 +188,7 @@ local function array_parser(keyword, element_type, ctype) return out end local elt_t = ctype and ffi.typeof(ctype) - local array_t = ctype and ffi.typeof('$[?]', ffi.typeof(elt_t)) + local array_t = ctype and ffi.typeof('$[?]', elt_t) local function finish(out) -- FIXME check min-elements if array_t then @@ -212,7 +212,7 @@ local function scalar_parser(keyword, argument_type, default, mandatory) end local function finish(out) if out ~= nil then return out end - if default then return parsev(default) end + if default then return parsev(default, keyword) end if mandatory then error('missing scalar value: '..keyword) end end return {init=init, parse=parse, finish=finish} @@ -420,7 +420,7 @@ local function data_printer_from_grammar(production) return function(data, file, indent) for _,v in ipairs(data) do print_keyword(keyword, file, indent) - file:write(serialize(data)) + file:write(serialize(v)) file:write(';\n') end end From 8f0c34bef43e1a9d59532bed253735b85256bfc3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 9 Nov 2016 18:36:13 +0100 Subject: [PATCH 185/631] Prune example lwaftr configs This runs the lwaftr configs that are part of Snabb through the parser and serializer again, which has the effect of not serializing default-valued configuration statements. --- .../lwaftr/tests/data/big_mtu_no_icmp.conf | 84 +- .../lwaftr/tests/data/icmp_on_fail.conf | 88 +- src/program/lwaftr/tests/data/no_hairpin.conf | 85 +- src/program/lwaftr/tests/data/no_icmp.conf | 86 +- .../lwaftr/tests/data/no_icmp_maxfrags1.conf | 85 +- .../data/no_icmp_with_filters_accept.conf | 86 +- .../no_icmp_with_filters_and_vlan_accept.conf | 86 +- .../no_icmp_with_filters_and_vlan_drop.conf | 86 +- .../tests/data/no_icmp_with_filters_drop.conf | 86 +- .../tests/data/small_ipv4_mtu_icmp.conf | 89 +- .../tests/data/small_ipv6_mtu_no_icmp.conf | 84 +- .../data/small_ipv6_mtu_no_icmp_allow.conf | 88 +- .../small_ipv6_mtu_no_icmp_vlan_allow.conf | 88 +- .../lwaftr/tests/data/tunnel_icmp.conf | 90 +- .../tests/data/tunnel_icmp_without_mac4.conf | 90 +- .../tests/data/tunnel_icmp_withoutmac.conf | 90 +- src/program/lwaftr/tests/data/vlan.conf | 90 +- .../tests/data/vlan/big_mtu_no_icmp.conf | 84 +- .../lwaftr/tests/data/vlan/icmp_on_fail.conf | 88 +- .../lwaftr/tests/data/vlan/no_hairpin.conf | 85 +- .../lwaftr/tests/data/vlan/no_icmp.conf | 86 +- .../tests/data/vlan/no_icmp_maxfrags1.conf | 85 +- .../vlan/no_icmp_with_filters_accept.conf | 86 +- .../data/vlan/no_icmp_with_filters_drop.conf | 86 +- .../tests/data/vlan/small_ipv4_mtu_icmp.conf | 89 +- .../data/vlan/small_ipv6_mtu_no_icmp.conf | 84 +- .../vlan/small_ipv6_mtu_no_icmp_allow.conf | 88 +- .../lwaftr/tests/data/vlan/tunnel_icmp.conf | 90 +- .../data/vlan/tunnel_icmp_without_mac4.conf | 90 +- .../data/vlan/tunnel_icmp_withoutmac.conf | 90 +- src/program/lwaftr/tests/data/vlan/vlan.conf | 90 +- src/program/lwaftr/virt/confs/lwaftrctl1.conf | 63 +- .../tests/conf/snabbvmx-lwaftr-xe0.conf | 1319 ++++++----------- .../snabbvmx/tests/conf/snabbvmx-lwaftr.conf | 1319 ++++++----------- .../end-to-end/data/snabbvmx-lwaftr-xe1.conf | 26 +- .../data/vlan/snabbvmx-lwaftr-xe1.conf | 26 +- 36 files changed, 1771 insertions(+), 3684 deletions(-) diff --git a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf index 033ee1cc28..2b72195ac0 100644 --- a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf @@ -5,123 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; @@ -132,17 +104,14 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; mtu 2000; @@ -151,6 +120,5 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/icmp_on_fail.conf b/src/program/lwaftr/tests/data/icmp_on_fail.conf index 403e4ac34b..9490b763e4 100644 --- a/src/program/lwaftr/tests/data/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/icmp_on_fail.conf @@ -5,152 +5,116 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/no_hairpin.conf b/src/program/lwaftr/tests/data/no_hairpin.conf index c0aa6ea8a3..139ddd328c 100644 --- a/src/program/lwaftr/tests/data/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/no_hairpin.conf @@ -5,152 +5,119 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; hairpinning false; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/no_icmp.conf b/src/program/lwaftr/tests/data/no_icmp.conf index 89cbd401d9..72e5e0cf99 100644 --- a/src/program/lwaftr/tests/data/no_icmp.conf +++ b/src/program/lwaftr/tests/data/no_icmp.conf @@ -5,152 +5,118 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf index a5cdf45ec2..63cc17d32d 100644 --- a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf @@ -5,147 +5,114 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 1; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf index 33eb8bd1de..9ab5faacee 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf @@ -5,116 +5,89 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { @@ -122,19 +95,16 @@ external-interface { egress-filter ip; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ingress-filter ip; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { @@ -142,19 +112,15 @@ internal-interface { egress-filter ip6; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ingress-filter ip6; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf index 7c918e85ec..ea3d03ea09 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf @@ -5,116 +5,89 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { @@ -122,19 +95,16 @@ external-interface { egress-filter ip; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ingress-filter ip; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -143,20 +113,16 @@ internal-interface { egress-filter ip6; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ingress-filter ip6; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf index 9607c978fa..646e66db82 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf @@ -5,116 +5,89 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { @@ -122,19 +95,16 @@ external-interface { egress-filter "len < 0"; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ingress-filter "len < 0"; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -143,20 +113,16 @@ internal-interface { egress-filter "len < 0"; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ingress-filter "len < 0"; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf index 44c914d0f7..be1d7a02ca 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf @@ -5,116 +5,89 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { @@ -122,19 +95,16 @@ external-interface { egress-filter "len < 0"; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ingress-filter "len < 0"; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { @@ -142,19 +112,15 @@ internal-interface { egress-filter "len < 0"; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ingress-filter "len < 0"; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf index b563350552..7b3ef4e296 100644 --- a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf @@ -5,125 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; mtu 576; @@ -132,25 +102,18 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf index a4d9919346..4676b2a84f 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf @@ -5,123 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; @@ -132,17 +104,14 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; mtu 1280; @@ -151,6 +120,5 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf index 965536eb1a..1deadeba53 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf @@ -5,125 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; mtu 1500; @@ -132,17 +102,12 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; mtu 1280; @@ -151,6 +116,5 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf index 1776a5922f..7ab70a365c 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf @@ -5,125 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; mtu 1500; @@ -132,18 +102,13 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; mtu 1280; @@ -152,7 +117,6 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/tunnel_icmp.conf b/src/program/lwaftr/tests/data/tunnel_icmp.conf index d2d2033be1..461e92d7e0 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp.conf @@ -5,152 +5,114 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf index 0f3b0254eb..c5a2dd5e3c 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf @@ -5,152 +5,114 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { ip 4.5.6.7; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf index 23f18efe35..fb2d00fe97 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf @@ -5,152 +5,114 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { ip 8:9:a:b:4:3:2:1; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/lwaftr/tests/data/vlan.conf b/src/program/lwaftr/tests/data/vlan.conf index 5832b2ee52..d6e9733278 100644 --- a/src/program/lwaftr/tests/data/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan.conf @@ -5,154 +5,116 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 6000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf index 9fe8b194a4..ec05e9087c 100644 --- a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf @@ -5,123 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; @@ -132,7 +104,6 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -140,10 +111,8 @@ internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; mtu 2000; @@ -152,7 +121,6 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf index 6ae8c9d674..ca44ef6d99 100644 --- a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf @@ -5,134 +5,103 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -140,19 +109,14 @@ internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf index 9dee35ff40..5eebdb78bc 100644 --- a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf @@ -5,134 +5,104 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -140,19 +110,16 @@ internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; hairpinning false; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp.conf b/src/program/lwaftr/tests/data/vlan/no_icmp.conf index 7a87407ce7..24469e9e73 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp.conf @@ -5,134 +5,104 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -140,19 +110,15 @@ internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf index a521ebc979..d6629ac14f 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf @@ -5,134 +5,104 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 1; - max-packets 20000; } vlan-tag 1092; } @@ -140,13 +110,10 @@ internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf index 7c918e85ec..ea3d03ea09 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf @@ -5,116 +5,89 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { @@ -122,19 +95,16 @@ external-interface { egress-filter ip; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ingress-filter ip; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -143,20 +113,16 @@ internal-interface { egress-filter ip6; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ingress-filter ip6; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf index 9607c978fa..646e66db82 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf @@ -5,116 +5,89 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { @@ -122,19 +95,16 @@ external-interface { egress-filter "len < 0"; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ingress-filter "len < 0"; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -143,20 +113,16 @@ internal-interface { egress-filter "len < 0"; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ingress-filter "len < 0"; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf index 8b9c45aad6..74feec9219 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf @@ -5,125 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; mtu 576; @@ -132,27 +102,20 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf index c8beaa2fe9..c54940c80d 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf @@ -5,123 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.10.10.10; @@ -132,7 +104,6 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } @@ -140,10 +111,8 @@ internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; mtu 1280; @@ -152,7 +121,6 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf index 1776a5922f..7ab70a365c 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf @@ -5,125 +5,95 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; mtu 1500; @@ -132,18 +102,13 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; mtu 1280; @@ -152,7 +117,6 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf index d708bd3048..e51c702c6e 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf @@ -5,154 +5,116 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf index d39da3969f..a805111a9c 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf @@ -5,154 +5,116 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { ip 4.5.6.7; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf index 790ebc797a..79f90d86ea 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf @@ -5,154 +5,116 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { ip 8:9:a:b:4:3:2:1; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/vlan.conf b/src/program/lwaftr/tests/data/vlan/vlan.conf index 5832b2ee52..d6e9733278 100644 --- a/src/program/lwaftr/tests/data/vlan/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan/vlan.conf @@ -5,154 +5,116 @@ binding-table { psid-map { addr 178.79.150.15; psid-length 4; - reserved-ports-bit-count 0; shift 12; } psid-map { addr 178.79.150.233; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.1; psid-length 0; - reserved-ports-bit-count 0; shift 16; } psid-map { addr 178.79.150.2; psid-length 16; - reserved-ports-bit-count 0; shift 0; } psid-map { addr 178.79.150.3; psid-length 6; - reserved-ports-bit-count 0; shift 10; } - softwire { - ipv4 178.79.150.3; - padding 0; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } softwire { ipv4 178.79.150.233; - padding 0; - psid 22788; + psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.1; - padding 0; psid 0; b4-ipv6 127:10:20:30:40:50:60:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 4660; + psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; } softwire { ipv4 178.79.150.233; - padding 0; - psid 54192; + psid 7850; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.233; - padding 0; - psid 7850; + psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { - ipv4 178.79.150.15; - padding 0; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - br 0; + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; - padding 0; - psid 2300; + psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; - br 0; } softwire { ipv4 178.79.150.15; - padding 0; - psid 0; + psid 1; b4-ipv6 127:22:33:44:55:66:77:128; - br 0; } softwire { - ipv4 178.79.150.233; - padding 0; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - br 0; - } - softwire { - ipv4 178.79.150.2; - padding 0; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 10.10.10.10; mac 12:12:12:12:12:12; - mtu 1460; next-hop { mac 68:68:68:68:68:68; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 6000; - period 2; } - generate-icmp-errors true; - hairpinning true; ip 8:9:a:b:c:d:e:f; mac 22:22:22:22:22:22; - mtu 1500; next-hop { mac 44:44:44:44:44:44; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } diff --git a/src/program/lwaftr/virt/confs/lwaftrctl1.conf b/src/program/lwaftr/virt/confs/lwaftrctl1.conf index af732a9901..5d945217cd 100644 --- a/src/program/lwaftr/virt/confs/lwaftrctl1.conf +++ b/src/program/lwaftr/virt/confs/lwaftrctl1.conf @@ -1,41 +1,24 @@ #!/usr/bin/env bash - -# General - -SNABB_HOST_PATH=~/workspace/snabb_host -SNABB_HOST_LWAFTR=$SNABB_HOST_PATH/src/program/lwaftr - -# QEMU - -NAME=lwaftr1 -DISK=~/vm/vm_lwaftr1.qcow2 -MEM=1024M - -IFACE=mgmt0 -NETSCRIPT=$SNABB_HOST_LWAFTR/virt/setup_networks/lwaftr1.sh - -SHARED_LOCATION=~/workspace -VHU_SOCK1=/tmp/vh1a.sock -VHU_SOCK2=/tmp/vh1b.sock -VNC_DISPLAY=22 - -# VM - -QEMU_CORE=0 -VM_DIR=~/vm -VM_IP=10.21.21.2 -VM_USER=igalia # Must be in the list of sudoers - -# Guest - -SNABB_GUEST_PATH=/mnt/host/snabb_guest/src - -# SnabbNFV - -SNABBNFV_CORE1=1 -SNABBNFV_CONF1=$SNABB_HOST_LWAFTR/virt/ports/lwaftr1/a.cfg -SNABBNFV_PCI1=0000:02:00.0 - -SNABBNFV_CORE2=2 -SNABBNFV_CONF2=$SNABB_HOST_LWAFTR/virt/ports/lwaftr1/b.cfg -SNABBNFV_PCI2=0000:02:00.1 + ^ +lib/yang/parser.lua:44: :1:2: error: Invalid identifier +stack traceback: + core/main.lua:137: in function + [C]: in function 'error' + lib/yang/parser.lua:44: in function 'error' + lib/yang/parser.lua:195: in function 'parse_identifier' + lib/yang/parser.lua:208: in function 'parse_keyword' + lib/yang/parser.lua:243: in function 'parse_statement' + lib/yang/parser.lua:232: in function 'parse_statement_list' + lib/yang/parser.lua:220: in function 'parse_string' + lib/yang/data.lua:332: in function 'load_data_for_schema_by_name' + lib/yang/yang.lua:107: in function 'load_configuration' + ...m/lwaftr/migrate_configuration/migrate_configuration.lua:350: in function 'run' + program/lwaftr/lwaftr.lua:17: in function 'run' + core/main.lua:56: in function + [C]: in function 'xpcall' + core/main.lua:185: in main chunk + [C]: at 0x00453c60 + [C]: in function 'pcall' + core/startup.lua:3: in main chunk + [C]: in function 'require' + [string "require "core.startup""]:1: in main chunk diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf index 2fae04e4de..8f618f3f9f 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf @@ -4,1359 +4,975 @@ binding-table { addr 193.5.1.100; end-addr 193.5.1.102; psid-length 6; - reserved-ports-bit-count 0; shift 10; } softwire { ipv4 193.5.1.100; - padding 0; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ad; - br 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:a8; } softwire { ipv4 193.5.1.100; - padding 0; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:b1; - br 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:aa; } softwire { ipv4 193.5.1.101; - padding 0; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:c4; - br 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:f8; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:132; - br 0; + ipv4 193.5.1.101; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:ce; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:12c; - br 0; + ipv4 193.5.1.101; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:eb; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:109; - br 0; + ipv4 193.5.1.100; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:9e; } softwire { ipv4 193.5.1.101; - padding 0; psid 63; b4-ipv6 fc00:1:2:3:4:5:0:fb; - br 0; } softwire { ipv4 193.5.1.102; - padding 0; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:11d; - br 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:119; + } + softwire { + ipv4 193.5.1.101; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ed; + } + softwire { + ipv4 193.5.1.101; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:c5; } softwire { ipv4 193.5.1.102; - padding 0; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:111; - br 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:fe; } softwire { ipv4 193.5.1.100; - padding 0; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:a4; - br 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:b5; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:c3; - br 0; + ipv4 193.5.1.100; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:b0; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:100; - br 0; + ipv4 193.5.1.101; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:d6; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:137; - br 0; + ipv4 193.5.1.100; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:a2; } softwire { ipv4 193.5.1.101; - padding 0; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:e4; - br 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:e8; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:bf; - br 0; + ipv4 193.5.1.100; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:8e; } softwire { ipv4 193.5.1.101; - padding 0; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:d5; - br 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ec; } softwire { ipv4 193.5.1.102; - padding 0; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:133; - br 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:122; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:12d; - br 0; + ipv4 193.5.1.101; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:c1; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:9e; - br 0; + ipv4 193.5.1.102; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:121; } softwire { ipv4 193.5.1.101; - padding 0; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:f9; - br 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:d1; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:135; - br 0; + ipv4 193.5.1.101; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:ee; } softwire { ipv4 193.5.1.101; - padding 0; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:ef; - br 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:df; } softwire { ipv4 193.5.1.100; - padding 0; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:99; - br 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:b4; } softwire { ipv4 193.5.1.100; - padding 0; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:a9; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:d7; - br 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ab; } softwire { ipv4 193.5.1.100; - padding 0; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:a8; - br 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:b2; } softwire { ipv4 193.5.1.100; - padding 0; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:91; - br 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:b7; } softwire { ipv4 193.5.1.101; - padding 0; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:da; - br 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:f3; } softwire { ipv4 193.5.1.101; - padding 0; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:de; - br 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:bd; } softwire { ipv4 193.5.1.100; - padding 0; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:a6; - br 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:ba; } softwire { ipv4 193.5.1.100; - padding 0; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:a0; - br 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:9c; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:be; - br 0; + ipv4 193.5.1.102; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:116; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:105; - br 0; + ipv4 193.5.1.101; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:f2; } softwire { ipv4 193.5.1.102; - padding 0; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:10c; - br 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:113; } softwire { ipv4 193.5.1.100; - padding 0; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:b4; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:d3; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:c9; - br 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:88; } softwire { ipv4 193.5.1.100; - padding 0; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:81; - br 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:9f; } softwire { ipv4 193.5.1.102; - padding 0; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:fc; - br 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:129; + } + softwire { + ipv4 193.5.1.101; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:cc; } softwire { ipv4 193.5.1.102; - padding 0; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:101; - br 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:10e; } softwire { ipv4 193.5.1.102; - padding 0; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:11b; - br 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:13a; } softwire { ipv4 193.5.1.100; - padding 0; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:86; - br 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:8c; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:d2; - br 0; + ipv4 193.5.1.100; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:8a; } softwire { ipv4 193.5.1.102; - padding 0; psid 40; b4-ipv6 fc00:1:2:3:4:5:0:123; - br 0; } softwire { ipv4 193.5.1.102; - padding 0; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:122; - br 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:fd; } softwire { ipv4 193.5.1.102; - padding 0; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:12a; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:96; - br 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:110; } softwire { ipv4 193.5.1.101; - padding 0; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:e9; - br 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:cf; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:e3; - br 0; + ipv4 193.5.1.102; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:138; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:12e; - br 0; + ipv4 193.5.1.100; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:9d; } softwire { ipv4 193.5.1.102; - padding 0; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:126; - br 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:109; } softwire { ipv4 193.5.1.100; - padding 0; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:89; - br 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:96; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ed; - br 0; + ipv4 193.5.1.100; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:91; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:106; - br 0; + ipv4 193.5.1.100; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:b8; + } + softwire { + ipv4 193.5.1.102; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:104; } softwire { ipv4 193.5.1.101; - padding 0; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:d0; - br 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:d3; } softwire { ipv4 193.5.1.102; - padding 0; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:10e; - br 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:137; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:11f; - br 0; + ipv4 193.5.1.101; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:e7; } softwire { ipv4 193.5.1.102; - padding 0; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:103; - br 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:108; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:121; - br 0; + ipv4 193.5.1.101; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:dd; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:7f; - br 0; + ipv4 193.5.1.102; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:12a; } softwire { ipv4 193.5.1.102; - padding 0; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:104; - br 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:120; } softwire { ipv4 193.5.1.100; - padding 0; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:95; - br 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:83; } softwire { ipv4 193.5.1.100; - padding 0; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:bc; - br 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ad; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:f5; - br 0; + ipv4 193.5.1.100; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:9b; } softwire { ipv4 193.5.1.100; - padding 0; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:9a; - br 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:bb; + } + softwire { + ipv4 193.5.1.102; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:10b; } softwire { ipv4 193.5.1.100; - padding 0; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:97; - br 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:af; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:120; - br 0; + ipv4 193.5.1.101; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:e6; } softwire { ipv4 193.5.1.102; - padding 0; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:117; - br 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:118; } softwire { ipv4 193.5.1.100; - padding 0; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:88; - br 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:b1; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:ba; - br 0; + ipv4 193.5.1.102; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:133; } softwire { ipv4 193.5.1.100; - padding 0; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:7e; - br 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ae; } softwire { ipv4 193.5.1.101; - padding 0; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:c1; - br 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:f1; } softwire { ipv4 193.5.1.102; - padding 0; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:108; - br 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:134; + } + softwire { + ipv4 193.5.1.100; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:a1; } softwire { ipv4 193.5.1.102; - padding 0; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:125; - br 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:100; } softwire { ipv4 193.5.1.101; - padding 0; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ec; - br 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:db; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:94; - br 0; + ipv4 193.5.1.102; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:135; + } + softwire { + ipv4 193.5.1.102; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:11e; } softwire { ipv4 193.5.1.100; - padding 0; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:9b; - br 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:a7; + } + softwire { + ipv4 193.5.1.102; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:ff; } softwire { ipv4 193.5.1.100; - padding 0; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ab; - br 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:89; } softwire { ipv4 193.5.1.100; - padding 0; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:9f; - br 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:80; } softwire { ipv4 193.5.1.100; - padding 0; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:93; - br 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:84; } softwire { ipv4 193.5.1.102; - padding 0; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:13a; - br 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:126; } softwire { ipv4 193.5.1.101; - padding 0; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:c5; - br 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:c6; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:cc; - br 0; + ipv4 193.5.1.100; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:b9; } softwire { ipv4 193.5.1.101; - padding 0; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:df; - br 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:d2; } softwire { ipv4 193.5.1.101; - padding 0; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:f1; - br 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:dc; } softwire { ipv4 193.5.1.102; - padding 0; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:116; - br 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:112; } softwire { ipv4 193.5.1.102; - padding 0; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:131; - br 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:124; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:f8; - br 0; + ipv4 193.5.1.100; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:90; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:10a; - br 0; + ipv4 193.5.1.101; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:e1; } softwire { ipv4 193.5.1.101; - padding 0; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:d6; - br 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:d5; } softwire { ipv4 193.5.1.100; - padding 0; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:aa; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:db; - br 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:a3; } softwire { ipv4 193.5.1.100; - padding 0; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:8a; - br 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:94; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:c0; - br 0; + ipv4 193.5.1.100; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:a5; } softwire { ipv4 193.5.1.100; - padding 0; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:87; - br 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:7e; } softwire { ipv4 193.5.1.102; - padding 0; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:136; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:a7; - br 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:132; } softwire { ipv4 193.5.1.101; - padding 0; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:c2; - br 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:e4; } softwire { ipv4 193.5.1.102; - padding 0; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:134; - br 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:10a; } softwire { ipv4 193.5.1.101; - padding 0; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:cf; - br 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:c2; } softwire { ipv4 193.5.1.100; - padding 0; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:8e; - br 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:99; } softwire { ipv4 193.5.1.101; - padding 0; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:fa; - br 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:e5; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:9d; - br 0; + ipv4 193.5.1.101; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:c3; } softwire { ipv4 193.5.1.101; - padding 0; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:e5; - br 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:fa; } softwire { ipv4 193.5.1.102; - padding 0; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:fe; - br 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:128; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:b9; - br 0; + ipv4 193.5.1.102; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:11a; } softwire { ipv4 193.5.1.101; - padding 0; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:f3; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:af; - br 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:c9; } softwire { ipv4 193.5.1.101; - padding 0; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:f4; - br 0; - } - softwire { - ipv4 193.5.1.102; - padding 0; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:12f; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:b7; - br 0; - } - softwire { - ipv4 193.5.1.102; - padding 0; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:11c; - br 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:f7; } softwire { ipv4 193.5.1.101; - padding 0; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:d8; - br 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:f5; } softwire { ipv4 193.5.1.102; - padding 0; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:127; - br 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:106; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ae; - br 0; + ipv4 193.5.1.101; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ea; } softwire { ipv4 193.5.1.102; - padding 0; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:107; - br 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:11d; } softwire { ipv4 193.5.1.100; - padding 0; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:8b; - br 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:7f; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:92; - br 0; + ipv4 193.5.1.101; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:da; } softwire { ipv4 193.5.1.101; - padding 0; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:e6; - br 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:e9; } softwire { ipv4 193.5.1.101; - padding 0; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:d4; - br 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:f9; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:82; - br 0; + ipv4 193.5.1.102; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:10c; } softwire { ipv4 193.5.1.101; - padding 0; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:f2; - br 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:d9; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:8c; - br 0; + ipv4 193.5.1.102; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:11c; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:a5; - br 0; + ipv4 193.5.1.101; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:c8; } softwire { ipv4 193.5.1.101; - padding 0; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:f7; - br 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:c0; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:129; - br 0; + ipv4 193.5.1.101; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:f0; } softwire { ipv4 193.5.1.102; - padding 0; psid 18; b4-ipv6 fc00:1:2:3:4:5:0:10d; - br 0; + } + softwire { + ipv4 193.5.1.101; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:bf; } softwire { ipv4 193.5.1.102; - padding 0; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:110; - br 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:105; } softwire { ipv4 193.5.1.100; - padding 0; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:a1; - br 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:85; } softwire { ipv4 193.5.1.101; - padding 0; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:ce; - br 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:be; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:112; - br 0; + ipv4 193.5.1.100; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:a4; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:b2; - br 0; + ipv4 193.5.1.101; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:e0; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:ac; - br 0; + ipv4 193.5.1.102; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:11b; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:c7; - br 0; + ipv4 193.5.1.102; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:127; + } + softwire { + ipv4 193.5.1.102; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:12b; } softwire { ipv4 193.5.1.102; - padding 0; psid 20; b4-ipv6 fc00:1:2:3:4:5:0:10f; - br 0; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:bd; - br 0; + ipv4 193.5.1.100; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:8b; } softwire { ipv4 193.5.1.100; - padding 0; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:90; - br 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:95; } softwire { ipv4 193.5.1.102; - padding 0; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:12b; - br 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:12e; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:f0; - br 0; + ipv4 193.5.1.102; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:114; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:113; - br 0; + ipv4 193.5.1.100; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:87; } softwire { ipv4 193.5.1.100; - padding 0; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:b0; - br 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:93; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:dc; - br 0; + ipv4 193.5.1.102; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:103; } softwire { ipv4 193.5.1.102; - padding 0; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:11e; - br 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:111; } softwire { ipv4 193.5.1.101; - padding 0; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:ee; - br 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:f4; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:11a; - br 0; + ipv4 193.5.1.100; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:98; } softwire { ipv4 193.5.1.100; - padding 0; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:a2; - br 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:bc; } softwire { ipv4 193.5.1.102; - padding 0; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:fd; - br 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:115; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:f6; - br 0; + ipv4 193.5.1.100; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:b3; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:e2; - br 0; + ipv4 193.5.1.100; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:a9; } softwire { ipv4 193.5.1.102; - padding 0; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:124; - br 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:12d; } softwire { ipv4 193.5.1.100; - padding 0; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:83; - br 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:86; } softwire { ipv4 193.5.1.101; - padding 0; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:ca; - br 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:f6; } softwire { ipv4 193.5.1.100; - padding 0; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:b6; - br 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:8f; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:118; - br 0; + ipv4 193.5.1.101; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:d8; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:8d; - br 0; + ipv4 193.5.1.101; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:d7; } softwire { ipv4 193.5.1.102; - padding 0; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:139; - br 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:131; } softwire { ipv4 193.5.1.102; - padding 0; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:115; - br 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:136; } softwire { ipv4 193.5.1.100; - padding 0; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:b5; - br 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:a6; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:dd; - br 0; + ipv4 193.5.1.102; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:107; } softwire { ipv4 193.5.1.100; - padding 0; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:84; - br 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:a0; } softwire { ipv4 193.5.1.101; - padding 0; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:d1; - br 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:c7; } softwire { ipv4 193.5.1.100; - padding 0; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:8f; - br 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:ac; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:bb; - br 0; + ipv4 193.5.1.101; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:d4; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:98; - br 0; + ipv4 193.5.1.102; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:fc; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:80; - br 0; + ipv4 193.5.1.101; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:c4; } softwire { ipv4 193.5.1.101; - padding 0; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:c8; - br 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:ca; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:85; - br 0; + ipv4 193.5.1.101; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:cd; } softwire { ipv4 193.5.1.102; - padding 0; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:119; - br 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:102; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:ff; - br 0; + ipv4 193.5.1.101; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:ef; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:b3; - br 0; + ipv4 193.5.1.102; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:101; } softwire { ipv4 193.5.1.102; - padding 0; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:10b; - br 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:139; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:e1; - br 0; + ipv4 193.5.1.102; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:117; } softwire { ipv4 193.5.1.100; - padding 0; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:9c; - br 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:8d; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:e7; - br 0; + ipv4 193.5.1.100; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:9a; } softwire { ipv4 193.5.1.102; - padding 0; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:114; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:cb; - br 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:125; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:102; - br 0; + ipv4 193.5.1.100; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:92; } softwire { ipv4 193.5.1.102; - padding 0; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:128; - br 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:12c; } softwire { ipv4 193.5.1.100; - padding 0; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:a3; - br 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:b6; } softwire { ipv4 193.5.1.101; - padding 0; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ea; - br 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:d0; } softwire { ipv4 193.5.1.102; - padding 0; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:138; - br 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:12f; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:e0; - br 0; + ipv4 193.5.1.100; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:97; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:d9; - br 0; + ipv4 193.5.1.102; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:11f; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:eb; - br 0; + ipv4 193.5.1.100; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:81; } softwire { ipv4 193.5.1.101; - padding 0; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:e8; - br 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:de; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:130; - br 0; + ipv4 193.5.1.100; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:82; } softwire { ipv4 193.5.1.101; - padding 0; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:cd; - br 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:e2; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:b8; - br 0; + ipv4 193.5.1.101; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:cb; } softwire { ipv4 193.5.1.101; - padding 0; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:c6; - br 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:e3; + } + softwire { + ipv4 193.5.1.102; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:130; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.0.1.1; mac 02:aa:aa:aa:aa:aa; - mtu 1460; next-hop { mac 02:99:99:99:99:99; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip fc00::100; mac 02:aa:aa:aa:aa:aa; mtu 9500; @@ -1365,6 +981,5 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf index 2fae04e4de..8f618f3f9f 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf @@ -4,1359 +4,975 @@ binding-table { addr 193.5.1.100; end-addr 193.5.1.102; psid-length 6; - reserved-ports-bit-count 0; shift 10; } softwire { ipv4 193.5.1.100; - padding 0; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ad; - br 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:a8; } softwire { ipv4 193.5.1.100; - padding 0; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:b1; - br 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:aa; } softwire { ipv4 193.5.1.101; - padding 0; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:c4; - br 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:f8; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:132; - br 0; + ipv4 193.5.1.101; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:ce; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:12c; - br 0; + ipv4 193.5.1.101; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:eb; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:109; - br 0; + ipv4 193.5.1.100; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:9e; } softwire { ipv4 193.5.1.101; - padding 0; psid 63; b4-ipv6 fc00:1:2:3:4:5:0:fb; - br 0; } softwire { ipv4 193.5.1.102; - padding 0; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:11d; - br 0; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:119; + } + softwire { + ipv4 193.5.1.101; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ed; + } + softwire { + ipv4 193.5.1.101; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:c5; } softwire { ipv4 193.5.1.102; - padding 0; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:111; - br 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:fe; } softwire { ipv4 193.5.1.100; - padding 0; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:a4; - br 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:b5; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:c3; - br 0; + ipv4 193.5.1.100; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:b0; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:100; - br 0; + ipv4 193.5.1.101; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:d6; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:137; - br 0; + ipv4 193.5.1.100; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:a2; } softwire { ipv4 193.5.1.101; - padding 0; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:e4; - br 0; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:e8; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:bf; - br 0; + ipv4 193.5.1.100; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:8e; } softwire { ipv4 193.5.1.101; - padding 0; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:d5; - br 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ec; } softwire { ipv4 193.5.1.102; - padding 0; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:133; - br 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:122; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:12d; - br 0; + ipv4 193.5.1.101; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:c1; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:9e; - br 0; + ipv4 193.5.1.102; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:121; } softwire { ipv4 193.5.1.101; - padding 0; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:f9; - br 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:d1; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:135; - br 0; + ipv4 193.5.1.101; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:ee; } softwire { ipv4 193.5.1.101; - padding 0; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:ef; - br 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:df; } softwire { ipv4 193.5.1.100; - padding 0; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:99; - br 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:b4; } softwire { ipv4 193.5.1.100; - padding 0; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:a9; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:d7; - br 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ab; } softwire { ipv4 193.5.1.100; - padding 0; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:a8; - br 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:b2; } softwire { ipv4 193.5.1.100; - padding 0; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:91; - br 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:b7; } softwire { ipv4 193.5.1.101; - padding 0; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:da; - br 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:f3; } softwire { ipv4 193.5.1.101; - padding 0; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:de; - br 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:bd; } softwire { ipv4 193.5.1.100; - padding 0; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:a6; - br 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:ba; } softwire { ipv4 193.5.1.100; - padding 0; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:a0; - br 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:9c; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:be; - br 0; + ipv4 193.5.1.102; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:116; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:105; - br 0; + ipv4 193.5.1.101; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:f2; } softwire { ipv4 193.5.1.102; - padding 0; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:10c; - br 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:113; } softwire { ipv4 193.5.1.100; - padding 0; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:b4; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:d3; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:c9; - br 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:88; } softwire { ipv4 193.5.1.100; - padding 0; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:81; - br 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:9f; } softwire { ipv4 193.5.1.102; - padding 0; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:fc; - br 0; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:129; + } + softwire { + ipv4 193.5.1.101; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:cc; } softwire { ipv4 193.5.1.102; - padding 0; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:101; - br 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:10e; } softwire { ipv4 193.5.1.102; - padding 0; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:11b; - br 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:13a; } softwire { ipv4 193.5.1.100; - padding 0; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:86; - br 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:8c; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:d2; - br 0; + ipv4 193.5.1.100; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:8a; } softwire { ipv4 193.5.1.102; - padding 0; psid 40; b4-ipv6 fc00:1:2:3:4:5:0:123; - br 0; } softwire { ipv4 193.5.1.102; - padding 0; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:122; - br 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:fd; } softwire { ipv4 193.5.1.102; - padding 0; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:12a; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:96; - br 0; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:110; } softwire { ipv4 193.5.1.101; - padding 0; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:e9; - br 0; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:cf; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:e3; - br 0; + ipv4 193.5.1.102; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:138; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:12e; - br 0; + ipv4 193.5.1.100; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:9d; } softwire { ipv4 193.5.1.102; - padding 0; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:126; - br 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:109; } softwire { ipv4 193.5.1.100; - padding 0; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:89; - br 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:96; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ed; - br 0; + ipv4 193.5.1.100; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:91; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:106; - br 0; + ipv4 193.5.1.100; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:b8; + } + softwire { + ipv4 193.5.1.102; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:104; } softwire { ipv4 193.5.1.101; - padding 0; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:d0; - br 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:d3; } softwire { ipv4 193.5.1.102; - padding 0; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:10e; - br 0; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:137; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:11f; - br 0; + ipv4 193.5.1.101; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:e7; } softwire { ipv4 193.5.1.102; - padding 0; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:103; - br 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:108; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:121; - br 0; + ipv4 193.5.1.101; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:dd; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:7f; - br 0; + ipv4 193.5.1.102; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:12a; } softwire { ipv4 193.5.1.102; - padding 0; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:104; - br 0; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:120; } softwire { ipv4 193.5.1.100; - padding 0; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:95; - br 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:83; } softwire { ipv4 193.5.1.100; - padding 0; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:bc; - br 0; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ad; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:f5; - br 0; + ipv4 193.5.1.100; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:9b; } softwire { ipv4 193.5.1.100; - padding 0; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:9a; - br 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:bb; + } + softwire { + ipv4 193.5.1.102; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:10b; } softwire { ipv4 193.5.1.100; - padding 0; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:97; - br 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:af; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:120; - br 0; + ipv4 193.5.1.101; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:e6; } softwire { ipv4 193.5.1.102; - padding 0; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:117; - br 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:118; } softwire { ipv4 193.5.1.100; - padding 0; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:88; - br 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:b1; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:ba; - br 0; + ipv4 193.5.1.102; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:133; } softwire { ipv4 193.5.1.100; - padding 0; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:7e; - br 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ae; } softwire { ipv4 193.5.1.101; - padding 0; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:c1; - br 0; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:f1; } softwire { ipv4 193.5.1.102; - padding 0; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:108; - br 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:134; + } + softwire { + ipv4 193.5.1.100; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:a1; } softwire { ipv4 193.5.1.102; - padding 0; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:125; - br 0; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:100; } softwire { ipv4 193.5.1.101; - padding 0; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ec; - br 0; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:db; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:94; - br 0; + ipv4 193.5.1.102; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:135; + } + softwire { + ipv4 193.5.1.102; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:11e; } softwire { ipv4 193.5.1.100; - padding 0; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:9b; - br 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:a7; + } + softwire { + ipv4 193.5.1.102; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:ff; } softwire { ipv4 193.5.1.100; - padding 0; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ab; - br 0; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:89; } softwire { ipv4 193.5.1.100; - padding 0; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:9f; - br 0; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:80; } softwire { ipv4 193.5.1.100; - padding 0; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:93; - br 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:84; } softwire { ipv4 193.5.1.102; - padding 0; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:13a; - br 0; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:126; } softwire { ipv4 193.5.1.101; - padding 0; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:c5; - br 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:c6; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:cc; - br 0; + ipv4 193.5.1.100; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:b9; } softwire { ipv4 193.5.1.101; - padding 0; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:df; - br 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:d2; } softwire { ipv4 193.5.1.101; - padding 0; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:f1; - br 0; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:dc; } softwire { ipv4 193.5.1.102; - padding 0; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:116; - br 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:112; } softwire { ipv4 193.5.1.102; - padding 0; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:131; - br 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:124; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:f8; - br 0; + ipv4 193.5.1.100; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:90; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:10a; - br 0; + ipv4 193.5.1.101; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:e1; } softwire { ipv4 193.5.1.101; - padding 0; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:d6; - br 0; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:d5; } softwire { ipv4 193.5.1.100; - padding 0; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:aa; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:db; - br 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:a3; } softwire { ipv4 193.5.1.100; - padding 0; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:8a; - br 0; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:94; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:c0; - br 0; + ipv4 193.5.1.100; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:a5; } softwire { ipv4 193.5.1.100; - padding 0; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:87; - br 0; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:7e; } softwire { ipv4 193.5.1.102; - padding 0; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:136; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:a7; - br 0; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:132; } softwire { ipv4 193.5.1.101; - padding 0; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:c2; - br 0; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:e4; } softwire { ipv4 193.5.1.102; - padding 0; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:134; - br 0; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:10a; } softwire { ipv4 193.5.1.101; - padding 0; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:cf; - br 0; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:c2; } softwire { ipv4 193.5.1.100; - padding 0; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:8e; - br 0; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:99; } softwire { ipv4 193.5.1.101; - padding 0; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:fa; - br 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:e5; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:9d; - br 0; + ipv4 193.5.1.101; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:c3; } softwire { ipv4 193.5.1.101; - padding 0; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:e5; - br 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:fa; } softwire { ipv4 193.5.1.102; - padding 0; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:fe; - br 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:128; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:b9; - br 0; + ipv4 193.5.1.102; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:11a; } softwire { ipv4 193.5.1.101; - padding 0; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:f3; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:af; - br 0; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:c9; } softwire { ipv4 193.5.1.101; - padding 0; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:f4; - br 0; - } - softwire { - ipv4 193.5.1.102; - padding 0; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:12f; - br 0; - } - softwire { - ipv4 193.5.1.100; - padding 0; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:b7; - br 0; - } - softwire { - ipv4 193.5.1.102; - padding 0; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:11c; - br 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:f7; } softwire { ipv4 193.5.1.101; - padding 0; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:d8; - br 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:f5; } softwire { ipv4 193.5.1.102; - padding 0; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:127; - br 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:106; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ae; - br 0; + ipv4 193.5.1.101; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ea; } softwire { ipv4 193.5.1.102; - padding 0; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:107; - br 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:11d; } softwire { ipv4 193.5.1.100; - padding 0; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:8b; - br 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:7f; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:92; - br 0; + ipv4 193.5.1.101; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:da; } softwire { ipv4 193.5.1.101; - padding 0; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:e6; - br 0; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:e9; } softwire { ipv4 193.5.1.101; - padding 0; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:d4; - br 0; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:f9; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:82; - br 0; + ipv4 193.5.1.102; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:10c; } softwire { ipv4 193.5.1.101; - padding 0; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:f2; - br 0; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:d9; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:8c; - br 0; + ipv4 193.5.1.102; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:11c; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:a5; - br 0; + ipv4 193.5.1.101; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:c8; } softwire { ipv4 193.5.1.101; - padding 0; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:f7; - br 0; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:c0; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:129; - br 0; + ipv4 193.5.1.101; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:f0; } softwire { ipv4 193.5.1.102; - padding 0; psid 18; b4-ipv6 fc00:1:2:3:4:5:0:10d; - br 0; + } + softwire { + ipv4 193.5.1.101; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:bf; } softwire { ipv4 193.5.1.102; - padding 0; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:110; - br 0; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:105; } softwire { ipv4 193.5.1.100; - padding 0; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:a1; - br 0; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:85; } softwire { ipv4 193.5.1.101; - padding 0; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:ce; - br 0; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:be; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:112; - br 0; + ipv4 193.5.1.100; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:a4; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:b2; - br 0; + ipv4 193.5.1.101; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:e0; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:ac; - br 0; + ipv4 193.5.1.102; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:11b; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:c7; - br 0; + ipv4 193.5.1.102; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:127; + } + softwire { + ipv4 193.5.1.102; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:12b; } softwire { ipv4 193.5.1.102; - padding 0; psid 20; b4-ipv6 fc00:1:2:3:4:5:0:10f; - br 0; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:bd; - br 0; + ipv4 193.5.1.100; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:8b; } softwire { ipv4 193.5.1.100; - padding 0; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:90; - br 0; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:95; } softwire { ipv4 193.5.1.102; - padding 0; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:12b; - br 0; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:12e; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:f0; - br 0; + ipv4 193.5.1.102; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:114; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:113; - br 0; + ipv4 193.5.1.100; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:87; } softwire { ipv4 193.5.1.100; - padding 0; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:b0; - br 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:93; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:dc; - br 0; + ipv4 193.5.1.102; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:103; } softwire { ipv4 193.5.1.102; - padding 0; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:11e; - br 0; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:111; } softwire { ipv4 193.5.1.101; - padding 0; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:ee; - br 0; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:f4; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:11a; - br 0; + ipv4 193.5.1.100; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:98; } softwire { ipv4 193.5.1.100; - padding 0; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:a2; - br 0; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:bc; } softwire { ipv4 193.5.1.102; - padding 0; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:fd; - br 0; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:115; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:f6; - br 0; + ipv4 193.5.1.100; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:b3; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:e2; - br 0; + ipv4 193.5.1.100; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:a9; } softwire { ipv4 193.5.1.102; - padding 0; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:124; - br 0; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:12d; } softwire { ipv4 193.5.1.100; - padding 0; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:83; - br 0; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:86; } softwire { ipv4 193.5.1.101; - padding 0; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:ca; - br 0; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:f6; } softwire { ipv4 193.5.1.100; - padding 0; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:b6; - br 0; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:8f; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:118; - br 0; + ipv4 193.5.1.101; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:d8; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:8d; - br 0; + ipv4 193.5.1.101; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:d7; } softwire { ipv4 193.5.1.102; - padding 0; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:139; - br 0; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:131; } softwire { ipv4 193.5.1.102; - padding 0; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:115; - br 0; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:136; } softwire { ipv4 193.5.1.100; - padding 0; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:b5; - br 0; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:a6; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:dd; - br 0; + ipv4 193.5.1.102; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:107; } softwire { ipv4 193.5.1.100; - padding 0; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:84; - br 0; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:a0; } softwire { ipv4 193.5.1.101; - padding 0; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:d1; - br 0; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:c7; } softwire { ipv4 193.5.1.100; - padding 0; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:8f; - br 0; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:ac; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:bb; - br 0; + ipv4 193.5.1.101; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:d4; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:98; - br 0; + ipv4 193.5.1.102; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:fc; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:80; - br 0; + ipv4 193.5.1.101; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:c4; } softwire { ipv4 193.5.1.101; - padding 0; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:c8; - br 0; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:ca; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:85; - br 0; + ipv4 193.5.1.101; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:cd; } softwire { ipv4 193.5.1.102; - padding 0; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:119; - br 0; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:102; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:ff; - br 0; + ipv4 193.5.1.101; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:ef; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:b3; - br 0; + ipv4 193.5.1.102; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:101; } softwire { ipv4 193.5.1.102; - padding 0; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:10b; - br 0; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:139; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:e1; - br 0; + ipv4 193.5.1.102; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:117; } softwire { ipv4 193.5.1.100; - padding 0; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:9c; - br 0; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:8d; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:e7; - br 0; + ipv4 193.5.1.100; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:9a; } softwire { ipv4 193.5.1.102; - padding 0; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:114; - br 0; - } - softwire { - ipv4 193.5.1.101; - padding 0; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:cb; - br 0; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:125; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:102; - br 0; + ipv4 193.5.1.100; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:92; } softwire { ipv4 193.5.1.102; - padding 0; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:128; - br 0; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:12c; } softwire { ipv4 193.5.1.100; - padding 0; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:a3; - br 0; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:b6; } softwire { ipv4 193.5.1.101; - padding 0; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ea; - br 0; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:d0; } softwire { ipv4 193.5.1.102; - padding 0; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:138; - br 0; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:12f; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:e0; - br 0; + ipv4 193.5.1.100; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:97; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:d9; - br 0; + ipv4 193.5.1.102; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:11f; } softwire { - ipv4 193.5.1.101; - padding 0; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:eb; - br 0; + ipv4 193.5.1.100; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:81; } softwire { ipv4 193.5.1.101; - padding 0; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:e8; - br 0; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:de; } softwire { - ipv4 193.5.1.102; - padding 0; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:130; - br 0; + ipv4 193.5.1.100; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:82; } softwire { ipv4 193.5.1.101; - padding 0; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:cd; - br 0; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:e2; } softwire { - ipv4 193.5.1.100; - padding 0; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:b8; - br 0; + ipv4 193.5.1.101; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:cb; } softwire { ipv4 193.5.1.101; - padding 0; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:c6; - br 0; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:e3; + } + softwire { + ipv4 193.5.1.102; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:130; } } external-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; ip 10.0.1.1; mac 02:aa:aa:aa:aa:aa; - mtu 1460; next-hop { mac 02:99:99:99:99:99; } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { allow-incoming-icmp false; error-rate-limiting { packets 600000; - period 2; } generate-icmp-errors false; - hairpinning true; ip fc00::100; mac 02:aa:aa:aa:aa:aa; mtu 9500; @@ -1365,6 +981,5 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf index 365c379f79..5382cf1772 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf @@ -3,52 +3,39 @@ binding-table { psid-map { addr 10.10.0.10; psid-length 6; - reserved-ports-bit-count 0; shift 10; } psid-map { addr 10.10.0.0; end-addr 10.10.0.1; psid-length 6; - reserved-ports-bit-count 0; shift 10; } softwire { ipv4 10.10.0.0; - padding 0; psid 1; b4-ipv6 2a02:587:f710::400; - br 0; } softwire { ipv4 10.10.0.0; - padding 0; psid 4; b4-ipv6 2a02:587:f710::430; - br 0; } softwire { ipv4 10.10.0.0; - padding 0; - psid 3; - b4-ipv6 2a02:587:f710::420; - br 0; + psid 2; + b4-ipv6 2a02:587:f710::410; } softwire { ipv4 10.10.0.0; - padding 0; - psid 2; - b4-ipv6 2a02:587:f710::410; - br 0; + psid 3; + b4-ipv6 2a02:587:f710::420; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 192.168.10.2; mac 02:cf:69:15:81:01; mtu 9000; @@ -57,16 +44,12 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; hairpinning false; ip fc00:168:10::2; mac 02:cf:69:15:81:01; @@ -76,6 +59,5 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } } diff --git a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf index 1f70ca971b..dd22d885b9 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf @@ -3,52 +3,39 @@ binding-table { psid-map { addr 10.10.0.10; psid-length 6; - reserved-ports-bit-count 0; shift 10; } psid-map { addr 10.10.0.0; end-addr 10.10.0.1; psid-length 6; - reserved-ports-bit-count 0; shift 10; } softwire { ipv4 10.10.0.0; - padding 0; psid 1; b4-ipv6 2a02:587:f710::400; - br 0; } softwire { ipv4 10.10.0.0; - padding 0; psid 4; b4-ipv6 2a02:587:f710::430; - br 0; } softwire { ipv4 10.10.0.0; - padding 0; - psid 3; - b4-ipv6 2a02:587:f710::420; - br 0; + psid 2; + b4-ipv6 2a02:587:f710::410; } softwire { ipv4 10.10.0.0; - padding 0; - psid 2; - b4-ipv6 2a02:587:f710::410; - br 0; + psid 3; + b4-ipv6 2a02:587:f710::420; } } external-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; ip 192.168.10.2; mac 02:cf:69:15:81:01; mtu 9000; @@ -57,17 +44,13 @@ external-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1092; } internal-interface { - allow-incoming-icmp true; error-rate-limiting { packets 600000; - period 2; } - generate-icmp-errors true; hairpinning false; ip fc00:168:10::2; mac 02:cf:69:15:81:01; @@ -77,7 +60,6 @@ internal-interface { } reassembly { max-fragments-per-packet 40; - max-packets 20000; } vlan-tag 1638; } From 0581a977f9df5c676e596ed1b4da1bdcd113af2b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 10 Nov 2016 16:37:22 +0100 Subject: [PATCH 186/631] Configuration refactor This PR refactors (re)configuration so that compute_config_actions results in an ordered list of actions, O(n) in the size of the change rather than of the graph, such that apply_config_actions can just mindly apply them. apply_config_actions is now a mutating procedure which mutates the app table, the link table, and the current configuration to reflect the result of applying the config diff in compute_config_actions. --- src/core/app.lua | 233 ++++++++++++++++++++++++++++------------------- 1 file changed, 140 insertions(+), 93 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 836524f62d..9c16c5a492 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -91,22 +91,29 @@ end function restart_dead_apps () if not use_restart then return end local restart_delay = 2 -- seconds - local actions = { start={}, restart={}, reconfig={}, keep={}, stop={} } - local restart = false + local actions = {} for name, app in pairs(app_table) do if app.dead and (now() - app.dead.time) >= restart_delay then - restart = true + local function add_action(action, ...) + table.insert(actions, { action, { ... } }) + end io.stderr:write(("Restarting %s (died at %f: %s)\n") - :format(app.appname, app.dead.time, app.dead.error)) - table.insert(actions.restart, app.appname) - else - table.insert(actions.keep, app.appname) + :format(name, app.dead.time, app.dead.error)) + add_action('stop_app', name) + add_action('start_app', name, + configuration.apps[name].class, + configuration.apps[name].arg) + for linkspec in pairs(configuration.links) do + local fa, fl, ta, tl = config.parse_link(linkspec) + if fa == name then add_action('link_output', fa, fl, linkspec) end + if ta == name then add_action('link_input', ta, tl, linkspec) end + end end end -- Restart dead apps if necessary. - if restart then apply_config_actions(actions, configuration) end + if #actions > 0 then apply_config_actions(actions) end end -- Configure the running app network to match new_configuration. @@ -115,61 +122,134 @@ end -- new app network by making the changes needed. function configure (new_config) local actions = compute_config_actions(configuration, new_config) - apply_config_actions(actions, new_config) - configuration = new_config + apply_config_actions(actions) counter.add(configs) end -- Return the configuration actions needed to migrate from old config to new. --- --- Here is an example return value for a case where two apps must --- start, one must stop, and one is kept as it is: --- { start = {'newapp1', 'newapp2'}, --- stop = {'deadapp1'}, --- keep = {'oldapp1'}, --- restart = {}, --- reconfig = {} --- } function compute_config_actions (old, new) - local actions = { start={}, restart={}, reconfig={}, keep={}, stop={} } + local actions = {} + local function add_action(action, ...) + table.insert(actions, { action, { ... } }) + end + + -- First determine the links that are going away and remove them. + for linkspec in pairs(old.links) do + if not new.links[linkspec] then + local fa, fl, ta, tl = config.parse_link(linkspec) + if old.apps[fa] then add_action('unlink_output', fa, fl) end + if old.apps[ta] then add_action('unlink_input', ta, tl) end + if not new[linkspec] then add_action('free_link', linkspec) end + end + end + + -- Do the same for apps. + for appname, info in pairs(old.apps) do + if not new.apps[appname] then add_action('stop_app', appname) end + end + + -- Start new apps, restart reclassed apps, or reconfigure apps with + -- changed configuration. + local fresh_apps = {} for appname, info in pairs(new.apps) do local class, arg = info.class, info.arg - local action = nil - if not old.apps[appname] then action = 'start' - elseif old.apps[appname].class ~= class then action = 'restart' - elseif not lib.equal(old.apps[appname].arg, arg) - then action = 'reconfig' - else action = 'keep' end - table.insert(actions[action], appname) - end - for appname in pairs(old.apps) do - if not new.apps[appname] then - table.insert(actions['stop'], appname) + if not old.apps[appname] then + add_action('start_app', appname, info.class, info.arg) + fresh_apps[appname] = true + elseif old.apps[appname].class ~= class then + add_action('stop_app', appname) + add_action('start_app', appname, info.class, info.arg) + fresh_apps[appname] = true + elseif not lib.equal(old.apps[appname].arg, arg) then + if class.reconfig then + add_action('reconfig_app', appname, info.arg) + else + add_action('stop_app', appname) + add_action('start_app', appname, info.class, info.arg) + fresh_apps[appname] = true + end + else + -- Otherwise if nothing changed, then nothing to do; we keep + -- the app around. + end + end + + -- Now rebuild links. + for linkspec in pairs(new.links) do + local fa, fl, ta, tl = config.parse_link(linkspec) + local fresh_link = not old.links[linkspec] + if fresh_link then add_action('new_link', linkspec) end + if not new.apps[fa] then error("no such app: " .. fa) end + if not new.apps[ta] then error("no such app: " .. ta) end + if fresh_link or fresh_apps[fa] then + add_action('link_output', fa, fl, linkspec) + end + if fresh_link or fresh_apps[ta] then + add_action('link_input', ta, tl, linkspec) end end + return actions end -- Update the active app network by applying the necessary actions. -function apply_config_actions (actions, conf) - -- The purpose of this function is to populate these tables: - local new_app_table, new_link_table = {}, {} +function apply_config_actions (actions) -- Table of functions that execute config actions local ops = {} - function ops.stop (name) - if app_table[name].stop then - app_table[name]:stop() - end - if app_table[name].shm then - shm.delete_frame(app_table[name].shm) + -- As an efficiency hack, some apps rely on the fact that we add + -- links both by name and by index to the "input" and "output" + -- objects. Probably they should be changed to just collect their + -- inputs and outputs in their :link() functions. Until then, call + -- this function when removing links from app input/output objects. + local function remove_link_from_array(array, link) + for i=1,#array do + if array[i] == link then + table.remove(array, i) + return + end end end - function ops.keep (name) - new_app_table[name] = app_table[name] - end - function ops.start (name) - local class = conf.apps[name].class - local arg = conf.apps[name].arg + function ops.unlink_output (appname, linkname) + local app = app_table[appname] + local link = app.output[linkname] + app.output[linkname] = nil + remove_link_from_array(app.output, link) + end + function ops.unlink_input (appname, linkname) + local app = app_table[appname] + local link = app.input[linkname] + app.input[linkname] = nil + remove_link_from_array(app.input, link) + end + function ops.free_link (linkspec) + link.free(link_table[linkspec], linkspec) + link_table[linkspec] = nil + configuration.links[linkspec] = nil + end + function ops.new_link (linkspec) + link_table[linkspec] = link.new(linkspec) + configuration.links[linkspec] = true + end + function ops.link_output (appname, linkname, linkspec) + local app = app_table[appname] + local link = assert(link_table[linkspec]) + app.output[linkname] = link + table.insert(app.output, link) + end + function ops.link_input (appname, linkname, linkspec) + local app = app_table[appname] + local link = assert(link_table[linkspec]) + app.input[linkname] = link + table.insert(app.input, link) + end + function ops.stop_app (name) + local app = app_table[name] + if app.stop then app:stop() end + if app.shm then shm.delete_frame(app.shm) end + app_table[name] = nil + configuration.apps[name] = nil + end + function ops.start_app (name, class, arg) local app = class:new(arg) if type(app) ~= 'table' then error(("bad return value from app '%s' start() method: %s"):format( @@ -179,61 +259,27 @@ function apply_config_actions (actions, conf) app.appname = name app.output = {} app.input = {} - new_app_table[name] = app + app_table[name] = app app.zone = zone if app.shm then app.shm.dtime = {counter, C.get_unix_time()} app.shm = shm.create_frame("apps/"..name, app.shm) end + configuration.apps[name] = { class = class, arg = arg } end - function ops.restart (name) - ops.stop(name) - ops.start(name) - end - function ops.reconfig (name) - if app_table[name].reconfig then - local arg = conf.apps[name].arg - local app = app_table[name] - app:reconfig(arg) - new_app_table[name] = app - else - ops.restart(name) - end - end - -- Dispatch actions in a suitable sequence. - for _, action in ipairs({'stop', 'restart', 'keep', 'reconfig', 'start'}) do - for _, name in ipairs(actions[action]) do - if log and action ~= 'keep' then - io.write("engine: ", action, " app ", name, "\n") - end - ops[action](name) - end + function ops.reconfig (name, arg) + local app = app_table[name] + app:reconfig(arg) + configuration.apps[name].arg = arg end - -- Setup links: create (or reuse) and renumber. - for linkspec in pairs(conf.links) do - local fa, fl, ta, tl = config.parse_link(linkspec) - if not new_app_table[fa] then error("no such app: " .. fa) end - if not new_app_table[ta] then error("no such app: " .. ta) end - -- Create or reuse a link and assign/update receiving app index - local link = link_table[linkspec] or link.new(linkspec) - -- Add link to apps - new_app_table[fa].output[fl] = link - table.insert(new_app_table[fa].output, link) - new_app_table[ta].input[tl] = link - table.insert(new_app_table[ta].input, link) - -- Remember link - new_link_table[linkspec] = link - end - -- Free obsolete links. - for linkspec, r in pairs(link_table) do - if not new_link_table[linkspec] then link.free(r, linkspec) end - end - -- Commit changes. - app_table, link_table = new_app_table, new_link_table - -- Trigger link event for each app. - for name, app in pairs(app_table) do - if app.link then app:link() end + + -- Dispatch actions. + for _, action in ipairs(actions) do + local name, args = unpack(action) + if log then io.write("engine: ", name, " ", args[1], "\n") end + assert(ops[name], name)(unpack(args)) end + compute_breathe_order () end @@ -483,6 +529,7 @@ function selftest () configure(c1) assert(app_table.app1 == orig_app1) assert(app_table.app2 == orig_app2) + assert(tostring(orig_link) == tostring(link_table['app1.x -> app2.x'])) local c2 = config.new() config.app(c2, "app1", App, "config") config.app(c2, "app2", App) From 37ebdbac6e0fcbd5b902cc08b8563f5cbc224d06 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Fri, 11 Nov 2016 09:20:52 +0100 Subject: [PATCH 187/631] Add explanation of more Hydra jobset parameters (#563) --- .../doc/README.continuous-integration.md | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/program/lwaftr/doc/README.continuous-integration.md b/src/program/lwaftr/doc/README.continuous-integration.md index e7a2ad478f..6a5e82f3a1 100644 --- a/src/program/lwaftr/doc/README.continuous-integration.md +++ b/src/program/lwaftr/doc/README.continuous-integration.md @@ -32,24 +32,35 @@ instance hosts the lwAftr CI benchmarks. Three jobsets are currently defined: ## Jobset parameters -(The following uses the `lwaftr-nic` jobset as an example.) +(The following uses the temporary `zzz-lwaftr-nic-dev` jobset as an example.) The parameters of each jobset are defined on its [configuration tab] -(https://hydra.snabb.co/jobset/igalia/lwaftr-nic#tabs-configuration). Each -lwAftr branch under test is pointed to a pair of parameters, named `snabbXname` -and `snabbXsrc`, where `X` is an uppercase letter. +(https://hydra.snabb.co/jobset/igalia/zzz-lwaftr-nic-dev#tabs-configuration). -A curve is drawn on each report graph for each `snabbXname/snabbXsrc` pair, -labeled with the `snabbXname` value, and showing benchmarks of the `snabbXsrc` -branch. +Each lwAftr branch under test is pointed to by a pair of parameters, named +`snabbXname` and `snabbXsrc`, where `X` is an uppercase letter. A curve is +drawn on each report graph for each `snabbXname/snabbXsrc` pair, labeled with +the `snabbXname` value, and showing benchmarks of the `snabbXsrc` branch. + +The `conf` parameter selects a configuration file from within the +`src/program/lwaftr/tests/data/` directory. + +The `ipv4PCap` and `ipv6PCap` parameters select data files from within the +`src/program/lwaftr/tests/data/` directory. + +The `duration` parameter states the amount of time, in seconds, that each test +will be run. + +The `times` parameter states how many times each test will be run. ## Reports Jobsets list executed jobs on the [Jobs tab] -(https://hydra.snabb.co/jobset/igalia/lwaftr-nic#tabs-jobs). Click on the -[reports.lwaftr](https://hydra.snabb.co/job/igalia/lwaftr-nic/reports.lwaftr) -job, and then on a successful build (indicated by a green mark). In the "Build -products" section, click on "report.html". +(https://hydra.snabb.co/jobset/igalia/zzz-lwaftr-nic-dev#tabs-jobs). Click on +the [reports.lwaftr] +(https://hydra.snabb.co/job/igalia/zzz-lwaftr-nic-dev/reports.lwaftr) job, and +then on a successful build (indicated by a green mark). In the "Build products" +section, click on "report.html". The report has three sections: From 49a84673bc84ffd26f82b3d9ffdd3b65fef6e19a Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Fri, 11 Nov 2016 11:06:30 +0000 Subject: [PATCH 188/631] core.worker: Remove CPU core as a parameter Narrows the scope of this module to only starting the worker process. The worker should setup any required affinity separately. --- src/core/worker.lua | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index 7878348c2e..fb418737a3 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -23,13 +23,10 @@ local function child (name) return children[name] or error("no such child: " .. name) end --- Start a worker process with affinity to a specific CPU core. --- The child will execute an app network when provided with configure(). -function start (name, core) +-- Start a worker to execute an app network when provided with configure(). +function start (name) local pid = S.fork() if pid == 0 then - -- Lock affinity for this child - S.sched_setaffinity(0, {core}) local env = { "SNABB_PROGRAM_LUACODE=require('core.worker').init()", "SNABB_WORKER_NAME="..name, "SNABB_WORKER_PARENT="..S.getppid() } @@ -37,7 +34,7 @@ function start (name, core) S.execve(("/proc/%d/exe"):format(S.getpid()), {}, env) else -- Parent process - children[name] = { pid = pid, core = core } + children[name] = { pid = pid } end end @@ -53,7 +50,6 @@ function status () local infop = S.waitid("pid", info.pid, "nohang, exited") status[name] = { pid = info.pid, - core = info.core, alive = infop.code == 0 } end @@ -127,8 +123,8 @@ function selftest () end print("Worker status:") for w, s in pairs(status()) do - print((" worker %s: pid=%s core=%s alive=%s"):format( - w, s.pid, s.core, s.alive)) + print((" worker %s: pid=%s alive=%s"):format( + w, s.pid, s.alive)) end print("Stopping children") for _, w in ipairs(workers) do @@ -137,8 +133,8 @@ function selftest () S.nanosleep(1) print("Worker status:") for w, s in pairs(status()) do - print((" worker %s: pid=%s core=%s alive=%s"):format( - w, s.pid, s.core, s.alive)) + print((" worker %s: pid=%s alive=%s"):format( + w, s.pid, s.alive)) end print("selftest: done") end From 6de0d3233955fb3a4b279b62d01972c9b99f2916 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 11 Nov 2016 14:42:07 +0100 Subject: [PATCH 189/631] Fix reconfiguration with reconfig method --- src/core/app.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/app.lua b/src/core/app.lua index 9c16c5a492..f2f24d4b26 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -267,7 +267,7 @@ function apply_config_actions (actions) end configuration.apps[name] = { class = class, arg = arg } end - function ops.reconfig (name, arg) + function ops.reconfig_app (name, arg) local app = app_table[name] app:reconfig(arg) configuration.apps[name].arg = arg From c04e8e1a59b519bf30825bc596efa9bb8b552b2b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 11 Nov 2016 15:30:56 +0100 Subject: [PATCH 190/631] Add apps.config.leader See https://github.com/snabbco/snabb/issues/1069. --- src/apps/config/leader.lua | 80 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/apps/config/leader.lua diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua new file mode 100644 index 0000000000..9b401ca8da --- /dev/null +++ b/src/apps/config/leader.lua @@ -0,0 +1,80 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +local S = require("syscall") +local ffi = require("ffi") +local yang = require("lib.yang.yang") + +Leader = { + config = { + socket_file_name = {required=true}, + -- worker_app_name = {required=true} + } +} + +function Leader:new (conf) + -- open socket + self.conf = conf + local socket = assert(S.socket("unix", "stream, nonblock")) + S.unlink(conf.socket_file_name) --unlink to avoid EINVAL on bind() + local sa = S.t.sockaddr_un(conf.socket_file_name) + assert(socket:bind(sa)) + assert(socket:listen()) + return setmetatable({socket=socket, peers={}}, {__index=Leader}) +end + +function Leader:pull () + local peers = self.peers + while true do + local sa = S.t.sockaddr_un() + local fd, err = self.socket:accept(sa) + if not fd then + if err.AGAIN then break end + assert(nil, err) + end + table.insert(peers, { state='length', read=0, accum='', fd=fd }) + end + local i = 1 + while i <= #peers do + local peer = peers[i] + if peer.state == 'length' then + --- + end + if peer.state == 'payload' then + --- + end + if peer.state == 'ready' then + --- + end + if peer.state == 'reply' then + --- + end + if peer.state == 'done' then + peer.fd:close() + table.remove(peers, i) + else + i = i + 1 + end + end +end + +function Leader:stop () + for _,peer in ipairs(self.peers) do peer.fd:close() end + self.peers = {} + self.socket:close() + S.unlink(self.conf.socket_file_name) +end + +function selftest () + print('selftest: apps.config.leader') + local pcap = require("apps.pcap.pcap") + local Match = require("apps.test.match").Match + local c = config.new() + local tmp = os.tmpname() + config.app(c, "leader", Leader, {socket_file_name=tmp}) + engine.configure(c) + engine.main({ duration = 0.0001, report = {showapps=true,showlinks=true}}) + os.remove(tmp) + print('selftest: ok') +end From 7a1503e5b7c4e4d434a5d2a5f973c87c56b22ddb Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 11 Nov 2016 16:14:43 +0100 Subject: [PATCH 191/631] Implement core state machine in config leader --- src/apps/config/leader.lua | 97 ++++++++++++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 9b401ca8da..d7193d2999 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -15,6 +15,7 @@ Leader = { function Leader:new (conf) -- open socket + S.signal('pipe', 'ign') self.conf = conf local socket = assert(S.socket("unix", "stream, nonblock")) S.unlink(conf.socket_file_name) --unlink to avoid EINVAL on bind() @@ -24,6 +25,11 @@ function Leader:new (conf) return setmetatable({socket=socket, peers={}}, {__index=Leader}) end +function Leader:handle(payload) + print('got a payload!', payload) + return payload +end + function Leader:pull () local peers = self.peers while true do @@ -33,26 +39,96 @@ function Leader:pull () if err.AGAIN then break end assert(nil, err) end - table.insert(peers, { state='length', read=0, accum='', fd=fd }) + fd:nonblock() + table.insert(peers, { state='length', len=0, fd=fd }) end local i = 1 while i <= #peers do local peer = peers[i] - if peer.state == 'length' then - --- + while peer.state == 'length' do + local ch, err = peer.fd:read(nil, 1) + if not ch then + if err.AGAIN then break end + peer.state = 'error' + peer.msg = tostring(err) + elseif ch == '\n' then + peer.pos = 0 + peer.buf = ffi.new('uint8_t[?]', peer.len) + peer.state = 'payload' + elseif tonumber(ch) then + peer.len = peer.len * 10 + tonumber(ch) + if peer.len > 1e8 then + peer.state = 'error' + peer.msg = 'length too long: '..peer.len + end + else + peer.state = 'error' + peer.msg = 'unexpected character: '..ch + end end - if peer.state == 'payload' then - --- + while peer.state == 'payload' do + if peer.pos == peer.len then + peer.state = 'ready' + peer.payload = ffi.string(peer.buf, peer.len) + peer.buf, peer.len = nil, nil + else + local count, err = peer.fd:read(peer.buf + peer.pos, + peer.len - peer.pos) + if not count then + if err.AGAIN then break end + peer.state = 'error' + peer.msg = tostring(err) + elseif count == 0 then + peer.state = 'error' + peer.msg = 'short read' + else + peer.pos = peer.pos + count + assert(peer.pos <= peer.len) + end + end end - if peer.state == 'ready' then - --- + while peer.state == 'ready' do + local success, reply = pcall(self.handle, self, peer.payload) + peer.payload = nil + if success then + assert(type(reply) == 'string') + reply = #reply..'\n'..reply + peer.state = 'reply' + peer.buf = ffi.new('uint8_t[?]', #reply, reply) + peer.pos = 0 + peer.len = #reply + else + peer.state = 'error' + peer.msg = reply + end end - if peer.state == 'reply' then - --- + while peer.state == 'reply' do + if peer.pos == peer.len then + peer.state = 'done' + peer.buf, peer.len = nil, nil + else + local count, err = peer.fd:write(peer.buf + peer.pos, + peer.len - peer.pos) + if not count then + if err.AGAIN then break end + peer.state = 'error' + peer.msg = tostring(err) + elseif count == 0 then + peer.state = 'error' + peer.msg = 'short write' + else + peer.pos = peer.pos + count + assert(peer.pos <= peer.len) + end + end end if peer.state == 'done' then peer.fd:close() table.remove(peers, i) + elseif peer.state == 'error' then + print('error: '..peer.msg) + peer.fd:close() + table.remove(peers, i) else i = i + 1 end @@ -74,7 +150,8 @@ function selftest () local tmp = os.tmpname() config.app(c, "leader", Leader, {socket_file_name=tmp}) engine.configure(c) - engine.main({ duration = 0.0001, report = {showapps=true,showlinks=true}}) + print(tmp) + engine.main({ duration = 100, report = {showapps=true,showlinks=true}}) os.remove(tmp) print('selftest: ok') end From a2dd717d6c0456b9370d24a1198dc2f299a5d2f1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 11 Nov 2016 16:57:23 +0100 Subject: [PATCH 192/631] First implementation of snabb config get-config --- src/program/config/README.inc | 1 + src/program/config/get_config/README.inc | 4 ++ src/program/config/get_config/get_config.lua | 64 ++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 src/program/config/get_config/README.inc create mode 100644 src/program/config/get_config/get_config.lua diff --git a/src/program/config/README.inc b/src/program/config/README.inc index ca3ccad08d..f49754795e 100644 --- a/src/program/config/README.inc +++ b/src/program/config/README.inc @@ -1,2 +1,3 @@ Usage: config derive-data-format [OPTIONS] [] + get-config SOCKET MSG diff --git a/src/program/config/get_config/README.inc b/src/program/config/get_config/README.inc new file mode 100644 index 0000000000..7bc84823e5 --- /dev/null +++ b/src/program/config/get_config/README.inc @@ -0,0 +1,4 @@ +Data Format Usage: + snabb config get-config SOCKET MSG + +foo diff --git a/src/program/config/get_config/get_config.lua b/src/program/config/get_config/get_config.lua new file mode 100644 index 0000000000..eabe92671d --- /dev/null +++ b/src/program/config/get_config/get_config.lua @@ -0,0 +1,64 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local S = require("syscall") +local ffi = require("ffi") +local lib = require("core.lib") +local schema = require("lib.yang.schema") +local yang_data = require("lib.yang.data") + +-- Number of spaces a tab should consist of when indenting config. +local tab_spaces = 2 + +local function show_usage(status) + print(require("program.config.get_config.README_inc")) + main.exit(status) +end + +local function parse_args(args) + local handlers = {} + handlers.h = function() show_usage(0) end + args = lib.dogetopt(args, handlers, "h", {help="h"}) + if #args ~= 2 then show_usage(1) end + return unpack(args) +end + +local function read_length(socket) + local len = 0 + while true do + local ch = assert(socket:read(nil, 1)) + if ch == '\n' then return len end + assert(tonumber(ch), 'not a number: '..ch) + len = len * 10 + tonumber(ch) + assert(len < 1e8, 'length too long: '..len) + end +end + +local function read_msg(socket, len) + local buf = ffi.new('uint8_t[?]', len) + local pos = 0 + while pos < len do + local count = assert(socket:read(buf+pos, len-pos)) + if count == 0 then error('short read') end + pos = pos + count + end + return ffi.string(buf, len) +end + +function run(args) + local socket_file_name, msg = parse_args(args) + + local socket = assert(S.socket("unix", "stream")) + + local sa = S.t.sockaddr_un(socket_file_name) + assert(socket:connect(sa)) + + socket:write(tostring(#msg)..'\n'..msg) + + local len = read_length(socket) + local msg = read_msg(socket, len) + print(msg) + + socket:close() + main.exit(0) +end From 30f2ee62e905076b26c3154197e7697c609dcd8c Mon Sep 17 00:00:00 2001 From: Christian Graf Date: Fri, 11 Nov 2016 18:26:51 +0100 Subject: [PATCH 193/631] added howto craft packets via scapy tool --- src/program/snabbvmx/doc/README.scapy.md | 324 +++++++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 src/program/snabbvmx/doc/README.scapy.md diff --git a/src/program/snabbvmx/doc/README.scapy.md b/src/program/snabbvmx/doc/README.scapy.md new file mode 100644 index 0000000000..896cebfce3 --- /dev/null +++ b/src/program/snabbvmx/doc/README.scapy.md @@ -0,0 +1,324 @@ +# crafting packets via scapy + +The scapy tool can be used to craft packets (http://www.secdev.org/projects/scapy/). +If using scapy, please make sure to run at minimum version 2.3.2 - versions below did not work smoothly with the lw4o6 packets. + +For below commands, please have scapy installed and started via "scapy". +Note: if you intend to send crafted packets to wire, start scapy via "sudo" or as root. + +## starting scapy +``` +lab@cgrafubuntu2:~/cg-ubuntu2/testing1.0$ scapy +INFO: Can't import python gnuplot wrapper . Won't be able to plot. +INFO: Can't import PyX. Won't be able to use psdump() or pdfdump(). +INFO: Can't import python Crypto lib. Won't be able to decrypt WEP. +INFO: Can't import python Crypto lib. Disabled certificate manipulation tools +Welcome to Scapy (2.3.2) +>>> +``` + +### define IP packet** +Scapy allows to craft any layer of a packet. Below variable a_ip defines an IP Header. Scapy will any required field prepopulate if not defined otherwise. +``` +a_ip=IP() +``` + +To see how the actual packets look like, several commands can be used. The list below is not a complete reference: + +- ls(pkt) have the list of fields values +- pkt.summary() for a one-line summary +- pkt.show() for a developped view of the packet +- pkt.show2() same as show but on the assembled packet (checksum is calculated, for instance) + +**several options to display the crafted packet** + +ls(pkt) +``` +>>> ls (a_ip) +version : BitField (4 bits) = 4 (4) +ihl : BitField (4 bits) = None (None) +tos : XByteField = 0 (0) +len : ShortField = None (None) +id : ShortField = 1 (1) +flags : FlagsField (3 bits) = 0 (0) +frag : BitField (13 bits) = 0 (0) +ttl : ByteField = 64 (64) +proto : ByteEnumField = 0 (0) +chksum : XShortField = None (None) +src : SourceIPField (Emph) = '127.0.0.1' (None) +dst : IPField (Emph) = '127.0.0.1' ('127.0.0.1') +options : PacketListField = [] ([]) +``` + +pkt.summary() +``` +>>> a_ip.summary() +'192.168.1.1 > 127.0.0.1 hopopt' +``` + +pkt.show() +``` +>>> a_ip.show() +###[ IP ]### + version= 4 + ihl= None + tos= 0x0 + len= None + id= 1 + flags= + frag= 0 + ttl= 64 + proto= hopopt + chksum= None + src= 192.168.1.1 + dst= 127.0.0.1 + \options\ + ``` + + pkt.show2() + ``` + >>> a_ip.show2() +###[ IP ]### + version= 4L + ihl= 5L + tos= 0x0 + len= 20 + id= 1 + flags= + frag= 0L + ttl= 64 + proto= hopopt + **chksum= 0x3a3f** + src= 192.168.1.1 + dst= 127.0.0.1 + \options\ + ``` + +## change packet fields +a_ip holds the complete IP-header. Based on the output below the "src" does contain the src-address. +Any part can be changed as seen: +``` +>>> a_ip.src='192.168.1.1' +>>> a_ip.show() +###[ IP ]### + version= 4 + ihl= None + tos= 0x0 + len= None + id= 1 + flags= + frag= 0 + ttl= 64 + proto= hopopt + chksum= None + src= 192.168.1.1 <<< got modified via a_ip.src=192.168.1.1 + dst= 127.0.0.1 + \options\ +``` + +## crafting complete ethernet-frame (ICMP echo-request) +Lw4o6 requires to to look into the UDP/TCP ports to define the software the packets belongs to. +For ICMP the icmp-id field is being looked up to assign the correct softwire. Below iptables show how the B4-device is src-nat'ing subscribers packet by with overwriting the src udp,tcp and icmp-id field. + +**exemplary iptables rules as done by the B4-device** +``` +-A POSTROUTING -o mytun -p tcp -j SNAT --to-source 193.5.1.2:1024-2047 +-A POSTROUTING -o mytun -p udp -j SNAT --to-source 193.5.1.2:1024-2047 +-A POSTROUTING -o mytun -p icmp -j SNAT --to-source 193.5.1.2:1024-2047 +``` + +**ICMP echo-request with icmp-id 1024** +In the initial example only the IP-header is created. Scapy allows as well to define all layers in a single step. +``` +>>> a_l2l3=Ether(src='90:e2:ba:94:2a:bc', dst='5c:45:27:15:a0:0d')/IP(src='10.0.1.100',dst='10.10.0.0')/ICMP(type=8, code=0, id=1024) + +>>> a_l2l3.show() +###[ Ethernet ]### + dst= 5c:45:27:15:a0:0d + src= 90:e2:ba:94:2a:bc + type= IPv4 +###[ IP ]### + version= 4 + ihl= None + tos= 0x0 + len= None + id= 1 + flags= + frag= 0 + ttl= 64 + proto= icmp + chksum= None + src= 10.0.1.100 + dst= 10.10.0.0 + \options\ +###[ ICMP ]### + type= echo-request + code= 0 + chksum= None + id= 0x400 + seq= 0x0 +>>> +``` + +## sending the frame to wire +Please make sure you have started scapy as root, otherwise sending the frame will lead into an error (error: [Errno 1] Operation not permitted). +Scapy provides a "send" and "sendp" function. Sendp requires to craft the l2-header accordingly and requires the interface to sent-out the packets. + +``` +>>> sendp(a_l2l3,iface='p4p1') +. +Sent 1 packets. +``` + +checking the result via tcpdump +``` +lab@cgrafubuntu2:~/cg-ubuntu2/testing1.0$ sudo tcpdump -n -i p4p1 -e +tcpdump: verbose output suppressed, use -v or -vv for full protocol decode +listening on p4p1, link-type EN10MB (Ethernet), capture size 262144 bytes +04:33:23.800977 90:e2:ba:94:2a:bc > 5c:45:27:15:a0:0d, ethertype IPv4 (0x0800), length 42: 10.0.1.100 > 10.10.0.0: ICMP echo request, id 1024, seq 0, length 8 +``` + +## crafting lw4o6 encapsulated packet + +scapy allows to craft as well the lw4o6 encapsulated packet. +This time a udp-packet with src-port 1024 is crafted: + +``` +lw4o6=Ether(src='90:e2:ba:94:2a:bc', dst='5c:45:27:15:a0:0d')/IPv6(src='2a02:587:f710::40',dst='2a02:587:f700::100')/IP(src='10.0.1.100',dst='10.10.0.0')/UDP(sport=1024) + +>>> lw4o6.summary() +'Ether / IPv6 / IP / UDP 10.0.1.100:1024 > 10.10.0.0:domain' + +>>> lw4o6.show() +###[ Ethernet ]### + dst= 5c:45:27:15:a0:0d + src= 90:e2:ba:94:2a:bc + type= IPv6 +###[ IPv6 ]### + version= 6 + tc= 0 + fl= 0 + plen= None + nh= IP + hlim= 64 + src= 2a02:587:f710::40 + dst= 2a02:587:f700::100 +###[ IP ]### + version= 4 + ihl= None + tos= 0x0 + len= None + id= 1 + flags= + frag= 0 + ttl= 64 + proto= udp + chksum= None + src= 10.0.1.100 + dst= 10.10.0.0 + \options\ +###[ UDP ]### + sport= 1024 + dport= domain + len= None + chksum= None + +``` + +sending this packet +``` +>>> sendp(lw4o6,iface='p4p1') +. +Sent 1 packets. +``` + +checking the result via tcpdump +``` +lab@cgrafubuntu2:~/cg-ubuntu2/testing1.0$ sudo tcpdump -n -i p4p1 -e +tcpdump: verbose output suppressed, use -v or -vv for full protocol decode +listening on p4p1, link-type EN10MB (Ethernet), capture size 262144 bytes +04:40:27.745410 90:e2:ba:94:2a:bc > 5c:45:27:15:a0:0d, ethertype IPv6 (0x86dd), length 82: 2a02:587:f710::40 > 2a02:587:f700::100: 10.0.1.100.1024 > 10.10.0.0.53: [|domain] +``` + + +## Fragmentation + +### sample python script to create IPv4 and lw4o6 frags +scapy allows to generate overlapping, overwriting, incomplete and just valid fragments. +Below code just generates plain non-overlapping IPv4 and IPv6 fragments. It is important to notice the added Fragmentheader, otherwise scapy will not fragment the packet. + +### writing directly to a pcap instead of sending to wire +Below script uses the pcapwriter function to write the fragmented packets directly into a pcap instead of sending them to wire. + +``` +lab@cgrafubuntu2:~/cg-ubuntu2/testing1.0$ cat send-v4-v6-frags.py +#!/usr/bin/python + +from scapy.all import * +payload="A"*500+"B"*500 + +pktdump = PcapWriter("scapy.pcap", append=True, sync=False) + +a_lw4o6=Ether(src='90:e2:ba:94:2a:bc', dst='5c:45:27:15:a0:0d')/IPv6(src='2a02:587:f710::40',dst='2a02:587:f700::100',nh=44)/IPv6ExtHdrFragment(nh=4)/IP(src='10.0.1.100',dst='10.10.0.0')/UDP(sport=1024,dport=2000)/payload +a_ipv4=Ether(src='90:e2:ba:94:2a:bc', dst='5c:45:27:15:a0:0d')/IP(src='10.0.1.100',dst='10.10.0.0')/ICMP(type=8, code=0, id=1024)/payload + +a_lw4o6.summary() +a_ipv4.summary() + +frags4=fragment(a_ipv4,fragsize=500) +frags6=fragment6(a_lw4o6,500) + + +# IPv4 +counter=1 +for fragment4 in frags4: + print "Packet no#"+str(counter) + print "===================================================" + fragment4.show() #displays each fragment + counter+=1 + #sendp(fragment4,iface='p4p1') <<< uncomment to sent to wire... + pktdump.write(fragment4) + +# IPv6 +counter=1 +for fragment6 in frags6: + print "Packet no#"+str(counter) + print "===================================================" + fragment6.show() #displays each fragment + counter+=1 + #sendp(fragment6,iface='p4p1') <<< uncomment to sent to wire... + pktdump.write(fragment6) + + +``` + +The resulting IPv6 packet: +``` +>>> a_lw4o6.summary() +'Ether / IPv6 / IPv6ExtHdrFragment / IP / UDP 10.0.1.100:1024 > 10.10.0.0:cisco_sccp / Raw' +``` + +The resulting IPv4 packet +``` +>>> a_ipv4.summary() +'Ether / IP / ICMP 10.0.1.100 > 10.10.0.0 echo-request 0 / Raw' +``` + +**verify generated IPv4 and IPv6 fragments** +When the above script is started, both the IPv4 packet and the IPv6 packet get fragmented and sent to wire: + +``` +lab@cgrafubuntu2:~/cg-ubuntu2/testing1.0$ tcpdump -n -e -r scapy.pcap +reading from file scapy.pcap, link-type EN10MB (Ethernet) +10:13:30.938939 90:e2:ba:94:2a:bc > 5c:45:27:15:a0:0d, ethertype IPv4 (0x0800), length 538: 10.0.1.100 > 10.10.0.0: ICMP echo request, id 1024, seq 0, length 504 +10:13:30.939553 90:e2:ba:94:2a:bc > 5c:45:27:15:a0:0d, ethertype IPv4 (0x0800), length 538: 10.0.1.100 > 10.10.0.0: ip-proto-1 +10:13:30.944438 90:e2:ba:94:2a:bc > 5c:45:27:15:a0:0d, ethertype IPv6 (0x86dd), length 494: 2a02:587:f710::40 > 2a02:587:f700::100: frag (0|432) truncated-ip - 596 bytes missing! 10.0.1.100.1024 > 10.10.0.0.2000: UDP, length 1000 +10:13:30.945226 90:e2:ba:94:2a:bc > 5c:45:27:15:a0:0d, ethertype IPv6 (0x86dd), length 494: 2a02:587:f710::40 > 2a02:587:f700::100: frag (432|432) +10:13:30.946022 90:e2:ba:94:2a:bc > 5c:45:27:15:a0:0d, ethertype IPv6 (0x86dd), length 226: 2a02:587:f710::40 > 2a02:587:f700::100: frag (864|164) +``` + +## usecase for scapy +Please see the README.troubleshooting.md. This doc requires pcap-files to run end-to-end tests. Scapy allows to generate such pcaps. + + + From 4915d1fb1dcdc00b1f5f40b5f55c52abd2153a4e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Sat, 12 Nov 2016 12:20:30 +0100 Subject: [PATCH 194/631] YANG schema support for RPCs and notifications --- src/lib/yang/schema.lua | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 9626af1f1f..14b83293bf 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -688,6 +688,14 @@ function resolve(schema, features) visit_top_level(node, env, 'extensions') visit_top_level(node, env, 'features') visit_top_level(node, env, 'identities') + for _,prop in ipairs({'rpcs', 'notifications'}) do + node[prop] = shallow_copy(node[prop]) + for k,v in pairs(node[prop]) do node[prop][k] = visit(v, env) end + end + end + if node.kind == 'rpc' then + if node.input then node.input = visit(node.input, env) end + if node.output then node.output = visit(node.output, env) end end for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do if not pcall(lookup, env, 'features', feature) then @@ -700,18 +708,21 @@ function resolve(schema, features) node.primitive_type = node.type.primitive_type end end - for k,v in pairs(node.body or {}) do - if v.kind == 'uses' then - -- Inline "grouping" into "uses". - local grouping = lookup_lazy(env, 'groupings', v.id) - node.body[k] = nil - for k,v in pairs(grouping.body) do - assert(not node.body[k], 'duplicate identifier: '..k) - node.body[k] = v + if node.body then + node.body = shallow_copy(node.body) + for k,v in pairs(node.body or {}) do + if v.kind == 'uses' then + -- Inline "grouping" into "uses". + local grouping = lookup_lazy(env, 'groupings', v.id) + node.body[k] = nil + for k,v in pairs(grouping.body) do + assert(not node.body[k], 'duplicate identifier: '..k) + node.body[k] = v + end + -- TODO: Handle refine and augment statements. + else + node.body[k] = visit(v, env) end - -- TODO: Handle refine and augment statements. - else - node.body[k] = visit(v, env) end end return node, env @@ -733,7 +744,9 @@ function resolve(schema, features) node = shallow_copy(node) local module_env = {env=env, prefixes={}, extensions={}, features={}, identities={}, typedefs={}, groupings={}} - local module_body = shallow_copy(node.body) + node.body = shallow_copy(node.body) + node.rpcs = shallow_copy(node.rpcs) + node.notifications = shallow_copy(node.notifications) for k,v in pairs(pop_prop(node, 'includes')) do local submodule = lookup(env, 'submodules', k) assert(submodule.belongs_to.id == node.id) @@ -743,7 +756,9 @@ function resolve(schema, features) include(module_env.identities, submodule_env.identities) include(module_env.typedefs, submodule_env.typedefs) include(module_env.groupings, submodule_env.groupings) - include(module_body, submodule.body) + include(node.body, submodule.body) + include(node.rpcs, submodule.rpcs) + include(node.notifications, submodule.notifications) end if node.prefix then assert(node.kind == 'module', node.kind) From a32210a77c2e265c1b5f4d50bb96f7931503115b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Sat, 12 Nov 2016 14:08:09 +0100 Subject: [PATCH 195/631] Support RPC sequences in parser/printer We add a "sequence" data type that simply contains an ordered sequence of productions. We use this for RPC call and response lists. --- src/lib/yang/data.lua | 83 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 2c8edaacff..cec6de596c 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -100,6 +100,25 @@ function data_grammar_from_schema(schema) return {type="struct", members=members, ctype=struct_ctype(members)} end +function rpc_grammar_from_schema(schema) + local grammar = {} + for _,prop in ipairs({'input', 'output'}) do + grammar[prop] = { type="sequence", members={} } + for k,rpc in pairs(schema.rpcs) do + grammar[prop].members[k] = data_grammar_from_schema(rpc[prop]) + end + end + return grammar +end + +function rpc_input_grammar_from_schema(schema) + return rpc_grammar_from_schema(schema).input +end + +function rpc_output_grammar_from_schema(schema) + return rpc_grammar_from_schema(schema).output +end + local function integer_type(min, max) return function(str, k) return util.tointeger(str, k, min, max) @@ -218,6 +237,24 @@ local function scalar_parser(keyword, argument_type, default, mandatory) return {init=init, parse=parse, finish=finish} end +local function sequence_parser(keyword, members) + local function init() return {} end + local function parse1(node) + local sub = assert(members[node.keyword], + 'unrecognized rpc: '..node.keyword) + local id = normalize_id(node.keyword) + return {id=id, data=sub.finish(sub.parse(node, sub.init()))} + end + local function parse(node, out) + table.insert(out, parse1(node)) + return out + end + local function finish(out) + return out + end + return {init=init, parse=parse, finish=finish} +end + local function ctable_builder(key_t, value_t) local res = ctable.new({ key_type=key_t, value_type=value_t }) local builder = {} @@ -326,6 +363,9 @@ function data_parser_from_grammar(production) return scalar_parser(keyword, production.argument_type, production.default, production.mandatory) end + function handlers.sequence(keyword, production) + return sequence_parser(keyword, visitn(production.members)) + end local parser = visit1('(top level)', production) return function(str, filename) @@ -343,6 +383,14 @@ function load_data_for_schema_by_name(schema_name, str, filename) return load_data_for_schema(schema, str, filename) end +function rpc_input_parser_from_schema(schema) + return data_parser_from_grammar(rpc_grammar_from_schema(schema).input) +end + +function rpc_output_parser_from_schema(schema) + return data_parser_from_grammar(rpc_grammar_from_schema(schema).output) +end + local function encode_yang_string(str) if str:match("^[^%s;{}\"'/]*$") then return str end local out = {} @@ -487,6 +535,15 @@ local function data_printer_from_grammar(production) end end end + function handlers.sequence(keyword, production) + local printers = {} + for k,v in pairs(production.members) do printers[k] = printer(k, v) end + return function(data, file, indent) + for _,elt in ipairs(data) do + assert(printers[assert(elt.id)])(elt.data, file, indent) + end + end + end local top_printer = body_printer(production.members) return function(data, file) @@ -495,6 +552,18 @@ local function data_printer_from_grammar(production) end end +function data_string_printer_from_grammar(production) + local printer = data_printer_from_grammar(production) + return function(data) + local file = {} + local out = {} + function file:write(str) table.insert(out, str) end + function file:flush(str) end + printer(data, file) + return table.concat(out) + end +end + function data_printer_from_schema(schema) return data_printer_from_grammar(data_grammar_from_schema(schema)) end @@ -508,6 +577,20 @@ function print_data_for_schema_by_name(schema_name, data, file) return print_data_for_schema(schema, data, file) end +function rpc_printer_from_grammar(production) + local printer = data_string_printer_from_grammar( + { type='struct', members = { rpcs=production } }) + return function(rpcs) return printer({rpcs=rpcs}) end +end + +function rpc_input_printer_from_schema(schema) + return rpc_printer_from_grammar(rpc_grammar_from_schema(schema).input) +end + +function rpc_output_printer_from_schema(schema) + return rpc_printer_from_grammar(rpc_grammar_from_schema(schema).output) +end + function selftest() print('selfcheck: lib.yang.data') local test_schema = schema.load_schema([[module fruit { From 48bb8f213d5b9c1c2f5e801fdeeeb306f096710b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Sat, 12 Nov 2016 14:09:11 +0100 Subject: [PATCH 196/631] snabb config get-config uses RPC mechanism We serialize the call and parse the response using the RPC yang facilities. --- src/lib/yang/snabb-config-leader-v1.yang | 25 +++++++++ src/program/config/get_config/README.inc | 17 ++++-- src/program/config/get_config/get_config.lua | 54 +++++++++++++++----- 3 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 src/lib/yang/snabb-config-leader-v1.yang diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang new file mode 100644 index 0000000000..a032adee1a --- /dev/null +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -0,0 +1,25 @@ +module snabb-config-leader-v1 { + namespace snabb:config-leader; + prefix config-leader; + + organization "Igalia, S.L."; + contact "Andy Wingo "; + description + "RPC interface for ConfigLeader Snabb app."; + + revision 2016-11-12 { + description + "Initial revision."; + } + + rpc get-config { + input { + leaf schema { type string; mandatory true; } + leaf revision { type string; } + leaf path { type string; default "/"; } + } + output { + leaf config { type string; } + } + } +} diff --git a/src/program/config/get_config/README.inc b/src/program/config/get_config/README.inc index 7bc84823e5..83bbfce3d8 100644 --- a/src/program/config/get_config/README.inc +++ b/src/program/config/get_config/README.inc @@ -1,4 +1,15 @@ -Data Format Usage: - snabb config get-config SOCKET MSG +Usage: + snabb config get-config [OPTIONS] ID PATH + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help -foo diff --git a/src/program/config/get_config/get_config.lua b/src/program/config/get_config/get_config.lua index eabe92671d..188db316c3 100644 --- a/src/program/config/get_config/get_config.lua +++ b/src/program/config/get_config/get_config.lua @@ -5,7 +5,8 @@ local S = require("syscall") local ffi = require("ffi") local lib = require("core.lib") local schema = require("lib.yang.schema") -local yang_data = require("lib.yang.data") +local yang = require("lib.yang.yang") +local data = require("lib.yang.data") -- Number of spaces a tab should consist of when indenting config. local tab_spaces = 2 @@ -16,11 +17,18 @@ local function show_usage(status) end local function parse_args(args) + local schema_name, revision_date local handlers = {} - handlers.h = function() show_usage(0) end - args = lib.dogetopt(args, handlers, "h", {help="h"}) - if #args ~= 2 then show_usage(1) end - return unpack(args) + function handlers.h() show_usage(0) end + function handlers.s(arg) schema_name = arg end + function handlers.r(arg) revision_date = arg end + args = lib.dogetopt(args, handlers, "h:s:r", + {help="h", ['schema-name']="s", schema="s", + ['revision-date']="r", revision="r"}) + if not schema_name then show_usage(1) end + if #args < 2 or #args > 2 then show_usage(1) end + local instance_id, path = unpack(args) + return schema_name, revision_date, instance_id, path end local function read_length(socket) @@ -45,20 +53,40 @@ local function read_msg(socket, len) return ffi.string(buf, len) end -function run(args) - local socket_file_name, msg = parse_args(args) +function prepare_rpc_list(rpcs) + local schema = yang.load_schema_by_name('snabb-config-leader-v1') + local print_input = data.rpc_input_printer_from_schema(schema) + local parse_output = data.rpc_output_parser_from_schema(schema) + return print_input(rpcs), parse_output +end - local socket = assert(S.socket("unix", "stream")) +function prepare_rpc(id, data) + local str, parse_responses = prepare_rpc_list({{id=id, data=data}}) + local function parse_response(str) + local responses = parse_responses(str) + assert(#responses == 1) + assert(responses[1].id == id) + return responses[1].data + end + return str, parse_response +end - local sa = S.t.sockaddr_un(socket_file_name) - assert(socket:connect(sa)) +function run(args) + local schema_name, revision_date, instance_id, path = parse_args(args) + local data = { schema = schema_name, revision = revision_date, path = path } + local message, parse_response = prepare_rpc('get-config', data) - socket:write(tostring(#msg)..'\n'..msg) + print(message) + local socket = assert(S.socket("unix", "stream")) + local sa = S.t.sockaddr_un(instance_id) + assert(socket:connect(sa)) + socket:write(tostring(#message)..'\n'..message) local len = read_length(socket) local msg = read_msg(socket, len) - print(msg) - socket:close() + + print(parse_response(msg)) + main.exit(0) end From 253bddfc0d4f7e16c498bc23715a6b540187ca25 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Sat, 12 Nov 2016 16:15:37 +0100 Subject: [PATCH 197/631] Add RPC support module --- src/lib/yang/data.lua | 15 +++++--- src/lib/yang/rpc.lua | 79 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 src/lib/yang/rpc.lua diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index cec6de596c..b4e317b89e 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -242,11 +242,13 @@ local function sequence_parser(keyword, members) local function parse1(node) local sub = assert(members[node.keyword], 'unrecognized rpc: '..node.keyword) - local id = normalize_id(node.keyword) - return {id=id, data=sub.finish(sub.parse(node, sub.init()))} + return {id=node.keyword, data=sub.finish(sub.parse(node, sub.init()))} end local function parse(node, out) - table.insert(out, parse1(node)) + assert(not node.keyword) -- ? + for _,node in ipairs(node.statements) do + table.insert(out, parse1(node)) + end return out end local function finish(out) @@ -537,10 +539,13 @@ local function data_printer_from_grammar(production) end function handlers.sequence(keyword, production) local printers = {} - for k,v in pairs(production.members) do printers[k] = printer(k, v) end + for k,v in pairs(production.members) do + printers[k] = printer(k, v) + end return function(data, file, indent) for _,elt in ipairs(data) do - assert(printers[assert(elt.id)])(elt.data, file, indent) + local id = assert(elt.id) + assert(printers[id])(elt.data, file, indent) end end end diff --git a/src/lib/yang/rpc.lua b/src/lib/yang/rpc.lua new file mode 100644 index 0000000000..8556c3dd47 --- /dev/null +++ b/src/lib/yang/rpc.lua @@ -0,0 +1,79 @@ +-- Use of this source code is governed by the Apache 2.0 license; see +-- COPYING. +module(..., package.seeall) + +local schema = require("lib.yang.schema") +local data = require("lib.yang.data") + +function prepare_callee(schema_name) + local schema = schema.load_schema_by_name(schema_name) + return { + parse_input = data.rpc_input_parser_from_schema(schema), + print_output = data.rpc_output_printer_from_schema(schema) + } +end + +function prepare_caller(schema_name) + local schema = schema.load_schema_by_name('snabb-config-leader-v1') + return { + print_input = data.rpc_input_printer_from_schema(schema), + parse_output = data.rpc_output_parser_from_schema(schema) + } +end + +function prepare_calls(caller, calls) + local str = caller.print_input(calls) + local function parse_responses(str) + local responses = caller.parse_output(str) + assert(#responses == #calls) + local stripped_responses = {} + for i=1,#calls do + assert(responses[i].id == calls[i].id) + table.insert(stripped_responses, responses[i].data) + end + return stripped_responses + end + return str, parse_responses +end + +function prepare_call(caller, id, data) + local str, parse_responses = prepare_calls(caller, {{id=id, data=data}}) + local function parse_response(str) return parse_responses(str)[1] end + return str, parse_response +end + +function handle_calls(callee, str, handle) + local responses = {} + for _,call in ipairs(callee.parse_input(str)) do + table.insert(responses, + { id=call.id, data=handle(call.id, call.data) }) + end + return callee.print_output(responses) +end + +function dispatch_handler(obj, prefix) + prefix = prefix or 'rpc_' + local normalize_id = data.normalize_id + return function(id, data) + local id = prefix..normalize_id(id) + local f = assert(obj[id], 'handler not found: '..id) + return f(obj, data) + end +end + +function selftest() + print('selftest: lib.yang.rpc') + local caller = prepare_caller('snabb-config-leader-v1') + local callee = prepare_callee('snabb-config-leader-v1') + local data = { schema = 'foo' } + local call_str, parse_response = prepare_call(caller, 'get-config', data) + local handler = {} + function handler:rpc_get_config(data) + return { config='pong '..data.schema } + end + local response_str = handle_calls(callee, call_str, + dispatch_handler(handler)) + local response = parse_response(response_str) + assert(response.config == 'pong foo') + print('selftest: ok') +end From 4a52f304d67ffb4236c2f823a296e0bb076a1137 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Sat, 12 Nov 2016 16:20:18 +0100 Subject: [PATCH 198/631] RPC between "snabb config get-config" and leader app --- src/apps/config/leader.lua | 33 +++++++++++------ src/program/config/get_config/get_config.lua | 37 ++++---------------- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index d7193d2999..57e6fe35a9 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -5,6 +5,7 @@ module(...,package.seeall) local S = require("syscall") local ffi = require("ffi") local yang = require("lib.yang.yang") +local rpc = require("lib.yang.rpc") Leader = { config = { @@ -13,21 +14,32 @@ Leader = { } } -function Leader:new (conf) - -- open socket +local function open_socket(file) S.signal('pipe', 'ign') - self.conf = conf local socket = assert(S.socket("unix", "stream, nonblock")) - S.unlink(conf.socket_file_name) --unlink to avoid EINVAL on bind() - local sa = S.t.sockaddr_un(conf.socket_file_name) + S.unlink(file) --unlink to avoid EINVAL on bind() + local sa = S.t.sockaddr_un(file) assert(socket:bind(sa)) assert(socket:listen()) - return setmetatable({socket=socket, peers={}}, {__index=Leader}) + return socket +end + +function Leader:new (conf) + local ret = {} + ret.conf = conf + ret.socket = open_socket(conf.socket_file_name) + ret.peers = {} + ret.rpc_callee = rpc.prepare_callee('snabb-config-leader-v1') + ret.rpc_handler = rpc.dispatch_handler(ret, 'rpc_') + return setmetatable(ret, {__index=Leader}) +end + +function Leader:rpc_get_config(data) + return { config = "hey!" } end function Leader:handle(payload) - print('got a payload!', payload) - return payload + return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end function Leader:pull () @@ -150,8 +162,7 @@ function selftest () local tmp = os.tmpname() config.app(c, "leader", Leader, {socket_file_name=tmp}) engine.configure(c) - print(tmp) - engine.main({ duration = 100, report = {showapps=true,showlinks=true}}) - os.remove(tmp) + engine.main({ duration = 0.0001, report = {showapps=true,showlinks=true}}) + engine.configure(config.new()) print('selftest: ok') end diff --git a/src/program/config/get_config/get_config.lua b/src/program/config/get_config/get_config.lua index 188db316c3..caaeba7877 100644 --- a/src/program/config/get_config/get_config.lua +++ b/src/program/config/get_config/get_config.lua @@ -4,9 +4,7 @@ module(..., package.seeall) local S = require("syscall") local ffi = require("ffi") local lib = require("core.lib") -local schema = require("lib.yang.schema") -local yang = require("lib.yang.yang") -local data = require("lib.yang.data") +local rpc = require("lib.yang.rpc") -- Number of spaces a tab should consist of when indenting config. local tab_spaces = 2 @@ -35,6 +33,7 @@ local function read_length(socket) local len = 0 while true do local ch = assert(socket:read(nil, 1)) + assert(ch ~= '', 'short read') if ch == '\n' then return len end assert(tonumber(ch), 'not a number: '..ch) len = len * 10 + tonumber(ch) @@ -53,40 +52,18 @@ local function read_msg(socket, len) return ffi.string(buf, len) end -function prepare_rpc_list(rpcs) - local schema = yang.load_schema_by_name('snabb-config-leader-v1') - local print_input = data.rpc_input_printer_from_schema(schema) - local parse_output = data.rpc_output_parser_from_schema(schema) - return print_input(rpcs), parse_output -end - -function prepare_rpc(id, data) - local str, parse_responses = prepare_rpc_list({{id=id, data=data}}) - local function parse_response(str) - local responses = parse_responses(str) - assert(#responses == 1) - assert(responses[1].id == id) - return responses[1].data - end - return str, parse_response -end - function run(args) local schema_name, revision_date, instance_id, path = parse_args(args) + local caller = rpc.prepare_caller('snabb-config-leader-v1') local data = { schema = schema_name, revision = revision_date, path = path } - local message, parse_response = prepare_rpc('get-config', data) - - print(message) - + local msg, parse = rpc.prepare_call(caller, 'get-config', data) local socket = assert(S.socket("unix", "stream")) local sa = S.t.sockaddr_un(instance_id) assert(socket:connect(sa)) - socket:write(tostring(#message)..'\n'..message) + socket:write(tostring(#msg)..'\n'..msg) local len = read_length(socket) - local msg = read_msg(socket, len) + msg = read_msg(socket, len) socket:close() - - print(parse_response(msg)) - + print(parse(msg).config) main.exit(0) end From db25fd64426c5f713a097b835fe1e9ed605fd7d0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 09:48:01 +0100 Subject: [PATCH 199/631] Config leader manages app graph --- src/apps/config/leader.lua | 44 +++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 57e6fe35a9..f02a306c0d 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -6,15 +6,19 @@ local S = require("syscall") local ffi = require("ffi") local yang = require("lib.yang.yang") local rpc = require("lib.yang.rpc") +local app = require("core.app") +local app_graph = require("core.config") Leader = { config = { socket_file_name = {required=true}, + setup_fn = {required=true}, + initial_configuration = {}, -- worker_app_name = {required=true} } } -local function open_socket(file) +local function open_socket (file) S.signal('pipe', 'ign') local socket = assert(S.socket("unix", "stream, nonblock")) S.unlink(file) --unlink to avoid EINVAL on bind() @@ -25,24 +29,47 @@ local function open_socket(file) end function Leader:new (conf) - local ret = {} - ret.conf = conf + local ret = setmetatable({}, {__index=Leader}) + ret.socket_file_name = conf.socket_file_name ret.socket = open_socket(conf.socket_file_name) ret.peers = {} + ret.setup_fn = conf.setup_fn + ret.current_app_graph = app_graph.new() + ret:reset_configuration(conf.initial_configuration) ret.rpc_callee = rpc.prepare_callee('snabb-config-leader-v1') ret.rpc_handler = rpc.dispatch_handler(ret, 'rpc_') - return setmetatable(ret, {__index=Leader}) + return ret end -function Leader:rpc_get_config(data) +function Leader:reset_configuration (configuration) + self:flush_config_action_queue() + local new_app_graph = self.setup_fn(configuration) + self.config_action_queue = app.compute_config_actions(self.current_app_graph, + new_app_graph) + self.current_app_graph = new_app_graph + self.current_configuration = configuration +end + +function Leader:flush_config_action_queue (configuration) + -- Once we switch to multiple processes, this will wait until all + -- followers have processed the actions. For now it just runs all + -- the actions. + if self.config_action_queue then + app.apply_config_actions(self.config_action_queue) + self.config_action_queue = nil + end +end + +function Leader:rpc_get_config (data) return { config = "hey!" } end -function Leader:handle(payload) +function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end function Leader:pull () + self:flush_config_action_queue() local peers = self.peers while true do local sa = S.t.sockaddr_un() @@ -151,7 +178,7 @@ function Leader:stop () for _,peer in ipairs(self.peers) do peer.fd:close() end self.peers = {} self.socket:close() - S.unlink(self.conf.socket_file_name) + S.unlink(self.socket_file_name) end function selftest () @@ -160,7 +187,8 @@ function selftest () local Match = require("apps.test.match").Match local c = config.new() local tmp = os.tmpname() - config.app(c, "leader", Leader, {socket_file_name=tmp}) + local function setup_fn(cfg) return app_graph.new() end + config.app(c, "leader", Leader, {socket_file_name=tmp, setup_fn=setup_fn}) engine.configure(c) engine.main({ duration = 0.0001, report = {showapps=true,showlinks=true}}) engine.configure(config.new()) From 58cdaa85128f11304112e554d573c805c36c2b44 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 10:38:30 +0100 Subject: [PATCH 200/631] Config leader claims $root/PID/config-leader-socket Adapt "snabb config get-config" as well. --- src/apps/config/leader.lua | 12 ++++++++---- src/program/config/get_config/get_config.lua | 12 ++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index f02a306c0d..28a5915493 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -7,11 +7,12 @@ local ffi = require("ffi") local yang = require("lib.yang.yang") local rpc = require("lib.yang.rpc") local app = require("core.app") +local shm = require("core.shm") local app_graph = require("core.config") Leader = { config = { - socket_file_name = {required=true}, + socket_file_name = {default='config-leader-socket'}, setup_fn = {required=true}, initial_configuration = {}, -- worker_app_name = {required=true} @@ -31,7 +32,11 @@ end function Leader:new (conf) local ret = setmetatable({}, {__index=Leader}) ret.socket_file_name = conf.socket_file_name - ret.socket = open_socket(conf.socket_file_name) + if not ret.socket_file_name:match('^/') then + local instance_dir = shm.root..'/'..tostring(S.getpid()) + ret.socket_file_name = instance_dir..'/'..ret.socket_file_name + end + ret.socket = open_socket(ret.socket_file_name) ret.peers = {} ret.setup_fn = conf.setup_fn ret.current_app_graph = app_graph.new() @@ -186,9 +191,8 @@ function selftest () local pcap = require("apps.pcap.pcap") local Match = require("apps.test.match").Match local c = config.new() - local tmp = os.tmpname() local function setup_fn(cfg) return app_graph.new() end - config.app(c, "leader", Leader, {socket_file_name=tmp, setup_fn=setup_fn}) + config.app(c, "leader", Leader, {setup_fn=setup_fn}) engine.configure(c) engine.main({ duration = 0.0001, report = {showapps=true,showlinks=true}}) engine.configure(config.new()) diff --git a/src/program/config/get_config/get_config.lua b/src/program/config/get_config/get_config.lua index caaeba7877..884e4ebd60 100644 --- a/src/program/config/get_config/get_config.lua +++ b/src/program/config/get_config/get_config.lua @@ -4,6 +4,7 @@ module(..., package.seeall) local S = require("syscall") local ffi = require("ffi") local lib = require("core.lib") +local shm = require("core.shm") local rpc = require("lib.yang.rpc") -- Number of spaces a tab should consist of when indenting config. @@ -58,8 +59,15 @@ function run(args) local data = { schema = schema_name, revision = revision_date, path = path } local msg, parse = rpc.prepare_call(caller, 'get-config', data) local socket = assert(S.socket("unix", "stream")) - local sa = S.t.sockaddr_un(instance_id) - assert(socket:connect(sa)) + local tail = instance_id..'/config-leader-socket' + local by_name = S.t.sockaddr_un(shm.root..'/by-name/'..tail) + local by_pid = S.t.sockaddr_un(shm.root..'/'..tail) + if not socket:connect(by_name) and not socket:connect(by_pid) then + io.stderr:write( + "Could not connect to config leader socket on Snabb instance '".. + instance_id.."'.\n") + main.exit(1) + end socket:write(tostring(#msg)..'\n'..msg) local len = read_length(socket) msg = read_msg(socket, len) From a968569613eb567291b76669b73931df9b6411ef Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 10:48:52 +0100 Subject: [PATCH 201/631] Add Hz parameter to config leader This lowers the rate at which we call into the kernel to accept new connections, allowing us to limit the overhead of the config leader. In the very short term we want to test the config leader in the same process as the data plane, which will have an associated overhead that we need to minimize. --- src/apps/config/leader.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 28a5915493..1ee32d3925 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -15,6 +15,7 @@ Leader = { socket_file_name = {default='config-leader-socket'}, setup_fn = {required=true}, initial_configuration = {}, + Hz = {default=100}, -- worker_app_name = {required=true} } } @@ -40,6 +41,8 @@ function Leader:new (conf) ret.peers = {} ret.setup_fn = conf.setup_fn ret.current_app_graph = app_graph.new() + ret.period = 1/conf.Hz + ret.next_time = app.now() ret:reset_configuration(conf.initial_configuration) ret.rpc_callee = rpc.prepare_callee('snabb-config-leader-v1') ret.rpc_handler = rpc.dispatch_handler(ret, 'rpc_') @@ -74,6 +77,8 @@ function Leader:handle (payload) end function Leader:pull () + if app.now() < self.next_time then return end + self.next_time = app.now() + self.period self:flush_config_action_queue() local peers = self.peers while true do From 501950840a1fb6e4a7ba897668b06bfb85368a10 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 11:10:14 +0100 Subject: [PATCH 202/631] snabb lwaftr bench uses config leader --- src/program/lwaftr/bench/bench.lua | 18 +++++++++++------- src/program/lwaftr/setup.lua | 12 ++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index fb16d9c800..8a2d99ba84 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -31,14 +31,18 @@ function run(args) local opts, conf_file, inv4_pcap, inv6_pcap = parse_args(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) - local c = config.new() - setup.load_bench(c, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') - app.configure(c) + local graph = config.new() + setup.with_leader(setup.load_bench, graph, conf, + inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + app.configure(graph) - local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) - csv:add_app('sinkv4', { 'input' }, { input=opts.hydra and 'decap' or 'Decap.' }) - csv:add_app('sinkv6', { 'input' }, { input=opts.hydra and 'encap' or 'Encap.' }) - csv:activate() + local function start_sampling() + local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) + csv:add_app('sinkv4', { 'input' }, { input=opts.hydra and 'decap' or 'Decap.' }) + csv:add_app('sinkv6', { 'input' }, { input=opts.hydra and 'encap' or 'Encap.' }) + csv:activate() + end + timer.activate(timer.new('spawn_csv_stats', start_sampling, 1e6)) app.busywait = true app.main({duration=opts.duration}) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 263113fa08..3f8b9b10dc 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -1,6 +1,7 @@ module(..., package.seeall) local config = require("core.config") +local leader = require("apps.config.leader") local Intel82599 = require("apps.intel.intel_app").Intel82599 local PcapFilter = require("apps.packet_filter.pcap_filter").PcapFilter local V4V6 = require("apps.lwaftr.V4V6").V4V6 @@ -444,3 +445,14 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) link_source(c, unpack(sources)) link_sink(c, unpack(sinks)) end + +function with_leader(f, graph, conf, ...) + local args = {...} + local function setup_fn(conf) + local graph = config.new() + f(graph, conf, unpack(args)) + return graph + end + config.app(graph, 'leader', leader.Leader, + { setup_fn = setup_fn, initial_configuration = conf }) +end From 03e2d26dd6bbad57cbf6634a08ae15f2e9c3185c Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 14 Nov 2016 12:35:02 +0100 Subject: [PATCH 203/631] Add XPath parser for xpath -> lua path This converts an XPath query which would be made against a yang model and converts it to a lua path. This path can the be used to easily resolve a node in a data tree. --- src/lib/yang/xpath.lua | 148 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 src/lib/yang/xpath.lua diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua new file mode 100644 index 0000000000..dc5c843b41 --- /dev/null +++ b/src/lib/yang/xpath.lua @@ -0,0 +1,148 @@ +-- -- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +-- This module can be used to parse a path based on a yang schema (or it's +-- derivative grammar) and produce a lua table which is a native lua way +-- of representing a path. The path provided is a subset of XPath supporting +-- named keys such as [addr=1.2.3.4] and also basic positional querying +-- for arrays e.g [position()=1] for the first element. +-- +-- The structure of the path is dependent on the type the node is, the +-- conversions are as follows: +-- +-- Scala fields: +-- A lua string of the member name +-- Struct fields: +-- A lua string of the member name +-- Array fields: +-- This is a table which has a "name" property specifying member +-- name and a "key" field which is a 1 based integer to specify the +-- position in the array. +-- Table fields: +-- This is a table which has a "name" property specifying member +-- name and has a "keys" (not key) property which is either: +-- - A string representing the key if the table is string keyed. +-- - A lua table with corrisponding leaf names as the key and the +-- value as the value. +module(..., package.seeall) + +local schema = require("lib.yang.schema") +local data = require("lib.yang.data") + +local function extract_parts(fragment) + local rtn = {query={}} + rtn.name = string.match(fragment, "([^%[]+)") + for k,v in string.gmatch(fragment, "%[([^=]+)=([^%]]+)%]") do + rtn.query[k] = v + end + return rtn +end + +local handlers = {} +function handlers.scalar(fragment, tree) + return fragment.name, tree +end +function handlers.struct(fragment, tree) + return fragment.name, tree.members +end +function handlers.table(fragment, tree) + if #tree.keys == 1 then + -- Check if the key is a string based key + k, v = pairs(tree.keys)() + if v.argument_type.primitive_type == "string" then + return {name=fragment.name, key=fragment.query[k]}, tree + end + else + return {name=fragment.name, keys=fragment.query}, tree + end +end +function handlers.array(fragment, tree) + local position = fragment.query["position()"] + return {name=fragment.name, key=tonumber(position)} +end +function handle(node_type, fragment, tree) + return assert(handlers[node_type], node_type)(fragment, tree) +end + +-- Gets the next item in the path returning the element and the remaining +-- path fragment. For example "router.routes.route" will return "router" +-- and "routes.route". If the end is reached it'll return nil. +function next_element(path) + return string.match(path, "([^/]+)/?(.*)") +end + +-- Converts an XPath path to a lua array consisting of path componants. +-- A path compent can then be resolved on a yang data tree: + +local function convert_path(path, grammar) + -- Extract head, check type and dispatch to handler. + local head, tail = next_element(path) + local parts = extract_parts(head) + local err = "Invalid path: "..parts.name + local node + if grammar.type == "table" then + if grammar.keys[head] == nil then + node = assert(grammar.values[parts.name], err) + else + node = grammar.keys[head] + end + else + node = assert(grammar[parts.name], err) + end + local element, node = handle(node.type, parts, node) + if tail ~= "" then + local rtn = convert_path(tail, node) + table.insert(rtn, 1, element) + return rtn + else + return {element} + end +end + + +-- Loads a module and converts the rest of the path. +function load_from_path(path) + -- First extract and load the module name then load it. + module_name, path = next_element(fragment) + scm = schema.load_schema_by_name(module_name) + grammar = data.data_grammar_for_schema(scm) + return module_name, convert_path(path, grammar.members) +end + +function selftest() + print("selftest: lib.yang.xpath") + local schema_src = [[module snabb-simple-router { + namespace snabb:simple-router; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + leaf active { type boolean; default true; } + leaf-list blocked-ips { type inet:ipv4-address; } + + container routes { + presence true; + list route { + key addr; + leaf addr { type inet:ipv4-address; mandatory true; } + leaf port { type uint8 { range 0..11; } mandatory true; } + } + }}]] + + local schemalib = require("lib.yang.schema") + local datalib = require("lib.yang.data") + local schema = schemalib.load_schema(schema_src, "xpath-test") + local grammar = datalib.data_grammar_from_schema(schema) + + -- Test path to lua path. + local path = convert_path("/routes/route[addr=1.2.3.4]/port", grammar.members) + + assert(path[1] == "routes") + assert(path[2].name == "route") + assert(path[2].keys) + assert(path[2].keys["addr"] == "1.2.3.4") + assert(path[3] == "port") + + local path = convert_path("/blocked-ips[position()=4]/", grammar.members) + assert(path[1].name == "blocked-ips") + assert(path[1].key == 4) +end From cd6ae3cea03df8a7f279920e70469b5666439757 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 14 Nov 2016 12:36:36 +0100 Subject: [PATCH 204/631] Remove unnecessary imports in selftest --- src/lib/yang/xpath.lua | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index dc5c843b41..9563882847 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -128,10 +128,8 @@ function selftest() } }}]] - local schemalib = require("lib.yang.schema") - local datalib = require("lib.yang.data") - local schema = schemalib.load_schema(schema_src, "xpath-test") - local grammar = datalib.data_grammar_from_schema(schema) + local scm = schema.load_schema(schema_src, "xpath-test") + local grammar = data.data_grammar_from_schema(scm) -- Test path to lua path. local path = convert_path("/routes/route[addr=1.2.3.4]/port", grammar.members) From afc8f2ac7172157f8845e2d54eb24958f2e504e7 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 15:03:12 +0100 Subject: [PATCH 205/631] Add support for decimal64 We use doubles because we want to represent these numbers on the Lua side as normal Lua numbers, and decimal64 is what the YANG spec gives us to do that. It's not a perfect match though. --- src/lib/yang/value.lua | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua index 43eca86bbc..4ffe1c30f2 100644 --- a/src/lib/yang/value.lua +++ b/src/lib/yang/value.lua @@ -66,7 +66,18 @@ function types.boolean.tostring(val) return tostring(val) end -types.decimal64 = unimplemented('decimal64') +-- FIXME: We lose precision by representing a decimal64 as a double. +types.decimal64 = {ctype='double'} +function types.decimal64.parse(str, what) + local str = assert(str, 'missing value for '..what) + return assert(tonumber(str), 'not a number: '..str) +end +function types.decimal64.tostring(val) + -- FIXME: Make sure we are not given too many digits after the + -- decimal point. + return tostring(val) +end + types.empty = unimplemented('empty') types.identityref = unimplemented('identityref') types['instance-identifier'] = unimplemented('instance-identifier') From dece919a52d48c4929d5fc0115066f808370a9e9 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 15:10:54 +0100 Subject: [PATCH 206/631] Add support for lib.equal comparing int64 --- src/core/lib.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib.lua b/src/core/lib.lua index 68f782a3dd..5f56904b4d 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -25,6 +25,8 @@ function equal (x, y) end return true elseif type(x) == 'cdata' then + if x == y then return true end + if ffi.typeof(x) ~= ffi.typeof(y) then return false end local size = ffi.sizeof(x) if ffi.sizeof(y) ~= size then return false end return C.memcmp(x, y, size) == 0 From 2b9d1b82505d297a8bf3281c70173c67f8e118c1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 15:11:24 +0100 Subject: [PATCH 207/631] Add ad-hoc data serialization via yang modules This will be used during the interim time where not all Snabb apps have associated Lua schemas. --- src/lib/yang/binary.lua | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index d293a6069b..b83aa85582 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -268,6 +268,56 @@ function compile_data_for_schema_by_name(schema_name, data, filename, source_mti data, filename, source_mtime) end +-- Hackily re-use the YANG serializer for Lua data consisting of tables, +-- ffi data, numbers, and strings. Truly a hack; to be removed in the +-- all-singing YANG future that we deserve where all data has an +-- associated schema. +local function ad_hoc_grammar_from_data(data) + if type(data) == 'table' then + local members = {} + for k,v in pairs(data) do + assert(type(k) == 'string') + members[k] = ad_hoc_grammar_from_data(v) + end + return {type='struct', members=members} + elseif type(data) == 'cdata' then + -- Hackety hack. + local ctype = tostring(ffi.typeof(data)):match('^ctype<(.*)>$') + if ctype:match('%*') then + error('pointer in ffi cdata cannot be serialized: '..ctype) + elseif ctype:match('%[') or ctype:match('^struct ') then + return {type='struct', members={}, ctype=ctype} + elseif ctype == 'uint64_t' then + return {type='scalar', argument_type={primitive_type='uint64'}} + elseif ctype == 'int64_t' then + return {type='scalar', argument_type={primitive_type='int64'}} + elseif ctype == 'double' or ctype == 'float' then + return {type='scalar', argument_type={primitive_type='decimal64'}} + elseif pcall(tonumber, data) then + local primitive_type = assert(ctype:match('^(.*)_t$')) + return {type='scalar', argument_type={primitive_type=primitive_type}} + else + error('unhandled ffi ctype: '..ctype) + end + elseif type(data) == 'number' then + return {type='scalar', argument_type={primitive_type='decimal64'}} + elseif type(data) == 'string' then + return {type='scalar', argument_type={primitive_type='string'}} + elseif type(data) == 'boolean' then + return {type='scalar', argument_type={primitive_type='boolean'}} + else + error('unhandled data type: '..type(data)) + end +end + +function compile_ad_hoc_lua_data_to_file(file_name, data) + local grammar = ad_hoc_grammar_from_data(data) + local emitter = data_emitter(grammar) + -- Empty string as schema name; a hack. + local compiler = data_compiler_from_grammar(emitter, '') + return compiler(data, file_name) +end + local function read_compiled_data(stream, strtab) local function read_string() return assert(strtab[stream:read_uint32()]) From 154c77fdabf3f66ee8bbb90e3941e4a34ca7ba2b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 15:58:34 +0100 Subject: [PATCH 208/631] reconfig_app action takes class arg This will allow us to determine when we can use a YANG schema to encode the argument when shipping a config to a follower. --- src/core/app.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index f640621d71..6d9c9882af 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -210,7 +210,7 @@ function compute_config_actions (old, new) fresh_apps[appname] = true elseif not lib.equal(old.apps[appname].arg, arg) then if class.reconfig then - add_action('reconfig_app', appname, info.arg) + add_action('reconfig_app', appname, info.class, info.arg) else add_action('stop_app', appname) add_action('start_app', appname, info.class, info.arg) @@ -315,7 +315,7 @@ function apply_config_actions (actions) end configuration.apps[name] = { class = class, arg = arg } end - function ops.reconfig_app (name, arg) + function ops.reconfig_app (name, class, arg) local app = app_table[name] app:reconfig(arg) configuration.apps[name].arg = arg From 11ad5b9ea7903906db91f6e4e0fffbcf2748ff39 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 15:21:50 +0100 Subject: [PATCH 209/631] Implement encoding for leader-follower protocol --- src/apps/config/action_queue.lua | 167 +++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 src/apps/config/action_queue.lua diff --git a/src/apps/config/action_queue.lua b/src/apps/config/action_queue.lua new file mode 100644 index 0000000000..bf536d0c0e --- /dev/null +++ b/src/apps/config/action_queue.lua @@ -0,0 +1,167 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +local S = require("syscall") +local lib = require("core.lib") +local ffi = require("ffi") +local yang = require("lib.yang.yang") +local binary = require("lib.yang.binary") +local shm = require("core.shm") + +local action_names = { 'unlink_output', 'unlink_input', 'free_link', + 'new_link', 'link_output', 'link_input', 'stop_app', + 'start_app', 'reconfig_app' } +local action_codes = {} +for i, name in ipairs(action_names) do action_codes[name] = i end + +local actions = {} + +function actions.unlink_output (codec, appname, linkname) + codec:string(appname) + codec:string(linkname) +end +function actions.unlink_input (codec, appname, linkname) + codec:string(appname) + codec:string(linkname) +end +function actions.free_link (codec, linkspec) + codec:string(linkspec) +end +function actions.new_link (codec, linkspec) + codec:string(linkspec) +end +function actions.link_output (codec, appname, linkname, linkspec) + codec:string(appname) + codec:string(linkname) + codec:string(linkspec) +end +function actions.link_input (codec, appname, linkname, linkspec) + codec:string(appname) + codec:string(linkname) + codec:string(linkspec) +end +function actions.stop_app (codec, name) + codec:string(name) +end +function actions.start_app (codec, name, class, arg) + codec:string(name) + codec:class(class) + codec:config(class, arg) +end +function actions.reconfig_app (codec, name, class, arg) + codec:string(name) + codec:config(class, arg) +end + +local public_names = {} +local function find_public_name(obj) + if public_names[obj] then return unpack(public_names[obj]) end + for modname, mod in pairs(package.loaded) do + for name, val in pairs(mod) do + if val == obj then + if type(val) == 'table' and type(val.new) == 'function' then + public_names[obj] = { modname, name } + return modname, name + end + end + end + end + error('could not determine public name for object: '..tostring(obj)) +end + +local lower_case = "abcdefghijklmnopqrstuvwxyz" +local upper_case = lower_case:upper() +local extra = "0123456789_-" +local alphabet = table.concat({lower_case, upper_case, extra}) +assert(#alphabet == 64) +local function random_file_name() + local f = io.open('/dev/urandom', 'rb') + -- 22 bytes, but we only use 2^6=64 bits from each byte, so total of + -- 132 bits of entropy. + local bytes = f:read(22) + assert(#bytes == 22) + f:close() + local out = {} + for i=1,#bytes do + table.insert(out, alphabet:byte(bytes:byte(i) % 64 + 1)) + end + local basename = string.char(unpack(out)) + return shm.root..'/'..tostring(S.getpid())..'/app-conf-'..basename +end + +local function encoder() + local encoder = { out = {} } + function encoder:uint32(len) + table.insert(self.out, ffi.new('uint32_t[1]', len)) + end + function encoder:string(str) + self:uint32(#str) + table.insert(self.out, ffi.new('uint8_t[?]', #str, str)) + end + function encoder:class(class) + local require_path, name = find_public_name(class) + encoder:string(require_path) + encoder:string(name) + end + function encoder:config(class, arg) + local file_name = random_file_name() + if class.yang_schema then + yang.compile_data_for_schema_by_name(class.yang_schema, arg, + file_name) + else + binary.compile_ad_hoc_lua_data_to_file(file_name, arg) + end + encoder:string(file_name) + end + function encoder:finish() + local size = 0 + for _,src in ipairs(self.out) do size = size + ffi.sizeof(src) end + local dst = ffi.new('uint8_t[?]', size) + local pos = 0 + for _,src in ipairs(self.out) do + ffi.copy(dst + pos, src, ffi.sizeof(src)) + pos = pos + ffi.sizeof(src) + end + return dst + end + return encoder +end + +local function encode_action(action) + local name, args = unpack(action) + local codec = encoder() + codec:uint32(assert(action_codes[name], name)) + assert(actions[name], name)(codec, unpack(args)) + return codec:finish() +end + +function selftest () + print('selftest: apps.config.action_queue') + local function serialize(data) + local tmp = random_file_name() + print('serializing to:', tmp) + binary.compile_ad_hoc_lua_data_to_file(tmp, data) + local loaded = binary.load_compiled_data_file(tmp) + assert(loaded.schema_name == '') + assert(lib.equal(data, loaded.data)) + os.remove(tmp) + end + serialize('foo') + serialize({foo='bar'}) + serialize({foo={qux='baz'}}) + serialize(1) + serialize(1LL) + local appname, linkname, linkspec = 'foo', 'bar', 'foo.a -> bar.q' + local class, arg = require('apps.basic.basic_apps').Tee, {} + encode_action({'unlink_output', {appname, linkname}}) + encode_action({'unlink_input', {appname, linkname}}) + encode_action({'free_link', {linkspec}}) + encode_action({'new_link', {linkspec}}) + encode_action({'link_output', {appname, linkname, linkspec}}) + encode_action({'link_input', {appname, linkname, linkspec}}) + encode_action({'stop_app', {appname}}) + encode_action({'start_app', {appname, class, arg}}) + encode_action({'reconfig_app', {appname, class, arg}}) + print('selftest: ok') +end From d39a36873151d683a844740ea66740e3d3191129 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 15:59:10 +0100 Subject: [PATCH 210/631] Implement decoder side of leader-follower protocol --- src/apps/config/action_queue.lua | 119 ++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 35 deletions(-) diff --git a/src/apps/config/action_queue.lua b/src/apps/config/action_queue.lua index bf536d0c0e..4cdc0157a8 100644 --- a/src/apps/config/action_queue.lua +++ b/src/apps/config/action_queue.lua @@ -18,40 +18,50 @@ for i, name in ipairs(action_names) do action_codes[name] = i end local actions = {} function actions.unlink_output (codec, appname, linkname) - codec:string(appname) - codec:string(linkname) + local appname = codec:string(appname) + local linkname = codec:string(linkname) + return codec:finish(appname, linkname) end function actions.unlink_input (codec, appname, linkname) - codec:string(appname) - codec:string(linkname) + local appname = codec:string(appname) + local linkname = codec:string(linkname) + return codec:finish(appname, linkname) end function actions.free_link (codec, linkspec) - codec:string(linkspec) + local linkspec = codec:string(linkspec) + return codec:finish(linkspec) end function actions.new_link (codec, linkspec) - codec:string(linkspec) + local linkspec = codec:string(linkspec) + return codec:finish(linkspec) end function actions.link_output (codec, appname, linkname, linkspec) - codec:string(appname) - codec:string(linkname) - codec:string(linkspec) + local appname = codec:string(appname) + local linkname = codec:string(linkname) + local linkspec = codec:string(linkspec) + return codec:finish(appname, linkname, linkspec) end function actions.link_input (codec, appname, linkname, linkspec) - codec:string(appname) - codec:string(linkname) - codec:string(linkspec) + local appname = codec:string(appname) + local linkname = codec:string(linkname) + local linkspec = codec:string(linkspec) + return codec:finish(appname, linkname, linkspec) end -function actions.stop_app (codec, name) - codec:string(name) +function actions.stop_app (codec, appname) + local appname = codec:string(appname) + return codec:finish(appname) end -function actions.start_app (codec, name, class, arg) - codec:string(name) - codec:class(class) - codec:config(class, arg) +function actions.start_app (codec, appname, class, arg) + local appname = codec:string(appname) + local _class = codec:class(class) + local config = codec:config(class, arg) + return codec:finish(appname, _class, config) end -function actions.reconfig_app (codec, name, class, arg) - codec:string(name) - codec:config(class, arg) +function actions.reconfig_app (codec, appname, class, arg) + local appname = codec:string(appname) + local _class = codec:class(class) + local config = codec:config(class, arg) + return codec:finish(appname, _class, config) end local public_names = {} @@ -101,8 +111,8 @@ local function encoder() end function encoder:class(class) local require_path, name = find_public_name(class) - encoder:string(require_path) - encoder:string(name) + self:string(require_path) + self:string(name) end function encoder:config(class, arg) local file_name = random_file_name() @@ -112,7 +122,7 @@ local function encoder() else binary.compile_ad_hoc_lua_data_to_file(file_name, arg) end - encoder:string(file_name) + self:string(file_name) end function encoder:finish() local size = 0 @@ -132,8 +142,42 @@ local function encode_action(action) local name, args = unpack(action) local codec = encoder() codec:uint32(assert(action_codes[name], name)) - assert(actions[name], name)(codec, unpack(args)) - return codec:finish() + return assert(actions[name], name)(codec, unpack(args)) +end + +local uint32_ptr_t = ffi.typeof('uint32_t*') +local function decoder(buf, len) + local decoder = { buf=buf, len=len, pos=0 } + function decoder:read(count) + local ret = self.buf + self.pos + self.pos = self.pos + count + assert(self.pos <= self.len) + return ret + end + function decoder:uint32() + return ffi.cast(uint32_ptr_t, self:read(4))[0] + end + function decoder:string() + local len = self:uint32() + return ffi.string(self:read(len), len) + end + function decoder:class() + local require_path, name = self:string(), self:string() + return assert(require(require_path)[name]) + end + function decoder:config() + return binary.load_compiled_data_file(self:string()).data + end + function decoder:finish(...) + return { ... } + end + return decoder +end + +local function decode_action(buf, len) + local codec = decoder(buf, len) + local name = assert(action_names[codec:uint32()]) + return { name, assert(actions[name], name)(codec) } end function selftest () @@ -152,16 +196,21 @@ function selftest () serialize({foo={qux='baz'}}) serialize(1) serialize(1LL) + local function test_action(action) + local encoded = encode_action(action) + local decoded = decode_action(encoded, ffi.sizeof(encoded)) + assert(lib.equal(action, decoded)) + end local appname, linkname, linkspec = 'foo', 'bar', 'foo.a -> bar.q' local class, arg = require('apps.basic.basic_apps').Tee, {} - encode_action({'unlink_output', {appname, linkname}}) - encode_action({'unlink_input', {appname, linkname}}) - encode_action({'free_link', {linkspec}}) - encode_action({'new_link', {linkspec}}) - encode_action({'link_output', {appname, linkname, linkspec}}) - encode_action({'link_input', {appname, linkname, linkspec}}) - encode_action({'stop_app', {appname}}) - encode_action({'start_app', {appname, class, arg}}) - encode_action({'reconfig_app', {appname, class, arg}}) + test_action({'unlink_output', {appname, linkname}}) + test_action({'unlink_input', {appname, linkname}}) + test_action({'free_link', {linkspec}}) + test_action({'new_link', {linkspec}}) + test_action({'link_output', {appname, linkname, linkspec}}) + test_action({'link_input', {appname, linkname, linkspec}}) + test_action({'stop_app', {appname}}) + test_action({'start_app', {appname, class, arg}}) + test_action({'reconfig_app', {appname, class, arg}}) print('selftest: ok') end From 4d39f82c60a9de175eebf784e009116498ac8169 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 14 Nov 2016 19:09:44 +0100 Subject: [PATCH 211/631] Implement channel for leader-follower messages --- src/apps/config/action_queue.lua | 10 +- src/apps/config/channel.lua | 230 +++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 src/apps/config/channel.lua diff --git a/src/apps/config/action_queue.lua b/src/apps/config/action_queue.lua index 4cdc0157a8..dd40e0e859 100644 --- a/src/apps/config/action_queue.lua +++ b/src/apps/config/action_queue.lua @@ -133,12 +133,12 @@ local function encoder() ffi.copy(dst + pos, src, ffi.sizeof(src)) pos = pos + ffi.sizeof(src) end - return dst + return dst, size end return encoder end -local function encode_action(action) +function encode_action(action) local name, args = unpack(action) local codec = encoder() codec:uint32(assert(action_codes[name], name)) @@ -174,7 +174,7 @@ local function decoder(buf, len) return decoder end -local function decode_action(buf, len) +function decode_action(buf, len) local codec = decoder(buf, len) local name = assert(action_names[codec:uint32()]) return { name, assert(actions[name], name)(codec) } @@ -197,8 +197,8 @@ function selftest () serialize(1) serialize(1LL) local function test_action(action) - local encoded = encode_action(action) - local decoded = decode_action(encoded, ffi.sizeof(encoded)) + local encoded, len = encode_action(action) + local decoded = decode_action(encoded, len) assert(lib.equal(action, decoded)) end local appname, linkname, linkspec = 'foo', 'bar', 'foo.a -> bar.q' diff --git a/src/apps/config/channel.lua b/src/apps/config/channel.lua new file mode 100644 index 0000000000..7d3024e2e4 --- /dev/null +++ b/src/apps/config/channel.lua @@ -0,0 +1,230 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +-- A channel is a ring buffer used by the config leader app to send +-- updates to a follower. Each follower has its own ring buffer and is +-- the only reader to the buffer. The config leader is the only writer +-- to these buffers also. The ring buffer is just bytes; putting a +-- message onto the buffer will write a header indicating the message +-- size, then the bytes of the message. The channel ring buffer is +-- mapped into shared memory. Access to a channel will never block or +-- cause a system call. + +local ffi = require('ffi') +local S = require("syscall") +local lib = require('core.lib') +local shm = require('core.shm') + +local ring_buffer_t = ffi.typeof([[struct { + uint32_t read; + uint32_t write; + uint32_t size; + uint8_t buf[?]; +}]]) + +-- Q: Why not just use shm.map? +-- A: We need a variable-sized mapping. +local function create_ring_buffer (name, size) + local path = shm.resolve(name) + shm.mkdir(path) + path = shm.root..'/'..path + local fd, err = S.open(path, "creat, rdwr, excl", '0664') + if not fd then + err = tostring(err or "unknown error") + error('error creating file "'..path..'": '..err) + end + local len = ffi.sizeof(ring_buffer_t, size) + assert(fd:ftruncate(len), "ring buffer: ftruncate failed") + local mem, err = S.mmap(nil, len, "read, write", "shared", fd, 0) + fd:close() + if mem == nil then error("mmap failed: " .. tostring(err)) end + mem = ffi.cast(ffi.typeof("$*", ring_buffer_t), mem) + ffi.gc(mem, function (ptr) S.munmap(ptr, len) end) + mem.size = size + return mem +end + +local function open_ring_buffer (name) + local path = shm.resolve(name) + local fd, err = S.open(path, "rdwr") + if not fd then + err = tostring(err or "unknown error") + error('error opening file "'..path..'": '..err) + end + local stat = S.fstat(fd) + local len = stat and stat.size + if len < ffi.sizeof(ring_buffer_t, 0) then + error("unexpected size for ring buffer") + end + local mem, err = S.mmap(nil, len, "read, write", "shared", fd, 0) + fd:close() + if mem == nil then error("mmap failed: " .. tostring(err)) end + mem = ffi.cast(ffi.typeof("$*", ring_buffer_t), mem) + ffi.gc(mem, function (ptr) S.munmap(ptr, len) end) + if len ~= ffi.sizeof(ring_buffer_t, mem.size) then + error("unexpected ring buffer size: "..tostring(len)) + end + return mem +end + +local function to_uint32 (num) + local buf = ffi.new('uint32_t[1]') + buf[0] = num + return buf[0] +end + +local function read_avail (ring) + lib.compiler_barrier() + return to_uint32(ring.write - ring.read) +end + +local function write_avail (ring) + return ring.size - read_avail(ring) +end + +Channel = {} + +-- Messages typically encode up to 3 or 4 strings like app names, link +-- names, module names, or the like. All of that and the length headers +-- probably fits within 256 bytes per message certainly. So make room +-- for around 4K messages, why not. +local default_buffer_size = 1024*1024 +function create(name, size) + local ret = {} + size = size or default_buffer_size + ret.ring_buffer = create_ring_buffer(name, size) + return setmetatable(ret, {__index=Channel}) +end + +function open(name) + local ret = {} + ret.ring_buffer = open_ring_buffer(name) + return setmetatable(ret, {__index=Channel}) +end + +-- The coordination needed between the reader and the writer is that: +-- +-- 1. If the reader sees a a bumped write pointer, that the data written +-- to the ring buffer will be available to the reader, i.e. the writer +-- has done whatever is needed to synchronize the data. +-- +-- 2. It should be possible for the reader to update the read pointer +-- without stompling other memory, notably the write pointer. +-- +-- 3. It should be possible for the writer to update the write pointer +-- without stompling other memory, notably the read pointer. +-- +-- 4. Updating a write pointer or a read pointer should eventually be +-- visible to the reader or writer, respectively. +-- +-- The full memory barrier after updates to the read or write pointer +-- ensures (1). The x86 memory model, and the memory model of C11, +-- guarantee (2) and (3). For (4), the memory barrier on the writer +-- side ensures that updates to the read or write pointers are +-- eventually visible to other CPUs, but we also have to insert a +-- compiler barrier before reading them to prevent LuaJIT from caching +-- their value somewhere else, like in a register. See +-- https://www.kernel.org/doc/Documentation/memory-barriers.txt for more +-- discussion on memory models, and +-- http://www.freelists.org/post/luajit/Compiler-loadstore-barrier-volatile-pointer-barriers-in-general,3 +-- for more on compiler barriers in LuaJIT. +-- +-- If there are multiple readers or writers, they should serialize their +-- accesses through some other mechanism. +-- + +-- Put some bytes onto the channel, but without updating the write +-- pointer. Precondition: the caller has checked that COUNT bytes are +-- indeed available for writing. +function Channel:put_bytes(bytes, count, offset) + offset = offset or 0 + local ring = self.ring_buffer + local start = (ring.write + offset) % ring.size + if start + count > ring.size then + local head = ring.size - start + ffi.copy(ring.buf + start, bytes, head) + ffi.copy(ring.buf, bytes + head, count - head) + else + ffi.copy(ring.buf + start, bytes, count) + end +end + +-- Peek some bytes into the channel. If the COUNT bytes are contiguous, +-- return a pointer into the channel. Otherwise allocate a buffer for +-- those bytes and return that. Precondition: the caller has checked +-- that COUNT bytes are indeed available for reading. +function Channel:peek_bytes(count, offset) + offset = offset or 0 + local ring = self.ring_buffer + local start = (ring.read + offset) % ring.size + local len + if start + count > ring.size then + local buf = ffi.new('uint8_t[?]', count) + local head_count = ring.size - start + local tail_count = count - head_count + ffi.copy(buf, ring.buf + start, head_count) + ffi.copy(buf + head_count, ring.buf, tail_count) + return buf + else + return ring.buf + start + end +end + +function Channel:put_message(bytes, count) + local ring = self.ring_buffer + if write_avail(ring) < count + 4 then return false end + self:put_bytes(ffi.cast('uint8_t*', ffi.new('uint32_t[1]', count)), 4) + self:put_bytes(bytes, count, 4) + ring.write = ring.write + count + 4 + ffi.C.full_memory_barrier() + return true; +end + +function Channel:peek_payload_len() + local ring = self.ring_buffer + local avail = read_avail(ring) + local count = 4 + if avail < count then return nil end + local len = ffi.cast('uint32_t*', self:peek_bytes(4))[0] + if avail < count + len then return nil end + return len +end + +function Channel:peek_message() + local payload_len = self:peek_payload_len() + if not payload_len then return nil, nil end + return self:peek_bytes(payload_len, 4), payload_len +end + +function Channel:discard_message(payload_len) + local ring = self.ring_buffer + ring.read = ring.read + payload_len + 4 + ffi.C.full_memory_barrier() +end + +function selftest() + print('selftest: apps.config.channel') + local msg_t = ffi.typeof('struct { uint8_t a; uint8_t b; }') + local ch = create('test/config-channel', (4+2)*16 + 1) + local function put(i) + return ch:put_message(ffi.new('uint8_t[2]', {i, i+16}), 2) + end + for _=1,4 do + for i=1,16 do assert(put(i)) end + assert(not put(17)) + local function assert_pop(i) + local msg, len = ch:peek_message() + assert(msg) + assert(len == 2) + assert(msg[0] == i) + assert(msg[1] == i + 16) + ch:discard_message(len) + end + assert_pop(1) + assert(put(17)) + for i=2,17 do assert_pop(i) end + assert(not ch:peek_message()) + end + print('selftest: channel ok') +end From 0b366dacc02d9149545d0606d2dc3fc76b36f818 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 15 Nov 2016 10:36:18 +0000 Subject: [PATCH 212/631] shm.alias: Ensure directory for alias exists --- src/core/shm.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/shm.lua b/src/core/shm.lua index c690972568..6c747bdb43 100644 --- a/src/core/shm.lua +++ b/src/core/shm.lua @@ -62,6 +62,7 @@ function exists (name) end function alias (name, target) + mkdir(lib.dirname(resolve(name))) assert(S.symlink(root.."/"..resolve(target), root.."/"..resolve(name)), "shm alias failed") end From 71897907f06c14e995e90e7faa9d54e93550f9e5 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 15 Nov 2016 10:36:36 +0000 Subject: [PATCH 213/631] core.main: Report unparsable SNABB_PROGRAM_LUACODE Prints a proper error message in this case rather than an obscure one. --- src/core/main.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/main.lua b/src/core/main.lua index ede9bde0f1..43cee38433 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -44,7 +44,12 @@ function main () if lib.getenv("SNABB_PROGRAM_LUACODE") then -- Run the given Lua code instead of the command-line local expr = lib.getenv("SNABB_PROGRAM_LUACODE") - loadstring(expr)() + local f = loadstring(expr) + if f == nil then + error(("Failed to load $SNABB_PROGRAM_LUACODE: %q"):format(expr)) + else + f() + end else -- Choose a program based on the command line local program, args = select_program(parse_command_line()) From 1ce4ca8e4a8254a393e87efbb0a6cf40e3b3afe0 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 15 Nov 2016 10:37:55 +0000 Subject: [PATCH 214/631] core.worker: Start worker with Lua expression The worker process now executes Lua code provided as a string to the start() function. This replaces the older API that assumed the parent would provide an app network config to the worker. The API to start a worker process is now: -- Start a named worker to execute the given Lua code (a string). start(name, luacode) Example: worker.start("w1", [[print("Hello from process w1")]]) --- src/core/worker.lua | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index fb418737a3..288066889c 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -23,14 +23,23 @@ local function child (name) return children[name] or error("no such child: " .. name) end --- Start a worker to execute an app network when provided with configure(). -function start (name) +-- Start a named worker to execute the given Lua code (a string). +function start (name, luacode) local pid = S.fork() if pid == 0 then - local env = { "SNABB_PROGRAM_LUACODE=require('core.worker').init()", - "SNABB_WORKER_NAME="..name, - "SNABB_WORKER_PARENT="..S.getppid() } + -- First we perform some initialization functions and then we + -- restart the process with execv(). + + -- Symlink the shm "group" folder to be shared via the parent process. + shm.alias("group", "/"..S.getppid().."/group") + -- Save the code we want to run in the environment. + S.setenv("SNABB_PROGRAM_LUACODE", luacode, true) + -- Restart the process with execve(). -- /proc/$$/exe is a link to the same Snabb executable that we are running + local env = {} + for key, value in pairs(S.environ()) do + table.insert(env, key.."="..value) + end S.execve(("/proc/%d/exe"):format(S.getpid()), {}, env) else -- Parent process @@ -50,7 +59,7 @@ function status () local infop = S.waitid("pid", info.pid, "nohang, exited") status[name] = { pid = info.pid, - alive = infop.code == 0 + alive = infop and infop.code == 0 or false } end return status @@ -74,7 +83,7 @@ end -- Initialize the worker by attaching to relevant shared memory -- objects and entering the main engine loop. -function init (name, parentpid) +function init () local name = assert(lib.getenv("SNABB_WORKER_NAME")) local parent = assert(lib.getenv("SNABB_WORKER_PARENT")) print(("Starting worker %s for parent %d"):format(name, parent)) @@ -82,20 +91,6 @@ function init (name, parentpid) -- Create "group" alias to the shared group folder in the parent process shm.alias("group", "/"..parent.."/group") - -- Wait for parent to provide an initial configuration - local warned - while not shm.exists("group/"..name.."/configs.counter") do - if not warned then - print("waiting for configuration...") - warned = true - S.nanosleep(0.001) - end - end - - -- Map the counter for how many times our configuration has been updated. - -- This provides an efficient way to poll for updates. - local configs = shm.map("group/"..name.."/configs", "counter") - -- Run the engine with continuous configuration updates local current_config local child_path = "group/config/..name" @@ -119,18 +114,20 @@ function selftest () local workers = { "w1", "w2", "w3" } print("Starting children") for _, w in ipairs(workers) do - start(w, 0) + start(w, ([[ print(" (hello world from worker %s. entering infinite loop...)") + while true do end -- infinite loop ]]):format(w)) end print("Worker status:") for w, s in pairs(status()) do print((" worker %s: pid=%s alive=%s"):format( w, s.pid, s.alive)) end + S.nanosleep(0.1) print("Stopping children") for _, w in ipairs(workers) do stop(w) end - S.nanosleep(1) + S.nanosleep(0.1) print("Worker status:") for w, s in pairs(status()) do print((" worker %s: pid=%s alive=%s"):format( From 5393afa803308436ee7971d22f3e6711957eea66 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 15 Nov 2016 10:43:31 +0000 Subject: [PATCH 215/631] core.worker: cleanup child w/ prctl set_pdeathsig Use prctl set_pdeathsig to automatically terminate the child (with an unhandled SIGHUP) upon death of the parent process. Comment notes that behavior with execve() may not be perfectly optimal. --- src/core/worker.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/worker.lua b/src/core/worker.lua index 288066889c..9769eccfd0 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -30,6 +30,13 @@ function start (name, luacode) -- First we perform some initialization functions and then we -- restart the process with execv(). + -- Terminate automatically when the parent dies. + -- + -- XXX This prctl setting needs to survive execve(). The Linux + -- execve(2) page seems to say that it will provided that the + -- binary being executed is not setuid or setgid. This may or + -- may not be adequate. + S.prctl("set_pdeathsig", "hup") -- Symlink the shm "group" folder to be shared via the parent process. shm.alias("group", "/"..S.getppid().."/group") -- Save the code we want to run in the environment. From f29a6a9b38e6dbc725461a38c4fbff374329b090 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 11:33:36 +0100 Subject: [PATCH 216/631] Implement follower; wire up leader-follower protocol --- src/apps/config/action_queue.lua | 13 ++-- src/apps/config/channel.lua | 1 + src/apps/config/follower.lua | 54 ++++++++++++++++ src/apps/config/leader.lua | 108 ++++++++++++++++++++++++------- src/apps/lwaftr/lwaftr.lua | 2 +- src/program/lwaftr/setup.lua | 5 +- 6 files changed, 152 insertions(+), 31 deletions(-) create mode 100644 src/apps/config/follower.lua diff --git a/src/apps/config/action_queue.lua b/src/apps/config/action_queue.lua index dd40e0e859..d764d50af4 100644 --- a/src/apps/config/action_queue.lua +++ b/src/apps/config/action_queue.lua @@ -68,11 +68,13 @@ local public_names = {} local function find_public_name(obj) if public_names[obj] then return unpack(public_names[obj]) end for modname, mod in pairs(package.loaded) do - for name, val in pairs(mod) do - if val == obj then - if type(val) == 'table' and type(val.new) == 'function' then - public_names[obj] = { modname, name } - return modname, name + if type(mod) == 'table' then + for name, val in pairs(mod) do + if val == obj then + if type(val) == 'table' and type(val.new) == 'function' then + public_names[obj] = { modname, name } + return modname, name + end end end end @@ -111,6 +113,7 @@ local function encoder() end function encoder:class(class) local require_path, name = find_public_name(class) + print('encoding', require_path, name) self:string(require_path) self:string(name) end diff --git a/src/apps/config/channel.lua b/src/apps/config/channel.lua index 7d3024e2e4..c13dcbd103 100644 --- a/src/apps/config/channel.lua +++ b/src/apps/config/channel.lua @@ -47,6 +47,7 @@ end local function open_ring_buffer (name) local path = shm.resolve(name) + path = shm.root..'/'..path local fd, err = S.open(path, "rdwr") if not fd then err = tostring(err or "unknown error") diff --git a/src/apps/config/follower.lua b/src/apps/config/follower.lua new file mode 100644 index 0000000000..28c438013b --- /dev/null +++ b/src/apps/config/follower.lua @@ -0,0 +1,54 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +local S = require("syscall") +local ffi = require("ffi") +local yang = require("lib.yang.yang") +local rpc = require("lib.yang.rpc") +local app = require("core.app") +local shm = require("core.shm") +local app_graph = require("core.config") +local channel = require("apps.config.channel") +local action_queue = require("apps.config.action_queue") + +Follower = { + config = { + Hz = {default=1000}, + } +} + +function Follower:new (conf) + local ret = setmetatable({}, {__index=Follower}) + ret.period = 1/conf.Hz + ret.next_time = app.now() + ret.channel = channel.create('config-follower-channel', 1e6) + return ret +end + +function Follower:handle_actions_from_leader() + local channel = self.channel + while true do + local buf, len = channel:peek_message() + if not buf then break end + local action = action_queue.decode_action(buf, len) + app.apply_config_actions({action}) + channel:discard_message(len) + end +end + +function Follower:pull () + if app.now() < self.next_time then return end + self.next_time = app.now() + self.period + self:handle_actions_from_leader() +end + +function selftest () + print('selftest: apps.config.follower') + local c = config.new() + config.app(c, "follower", Follower, {}) + engine.configure(c) + engine.main({ duration = 0.0001, report = {showapps=true,showlinks=true}}) + engine.configure(config.new()) + print('selftest: ok') +end diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 1ee32d3925..06883d40a2 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -9,14 +9,16 @@ local rpc = require("lib.yang.rpc") local app = require("core.app") local shm = require("core.shm") local app_graph = require("core.config") +local action_queue = require("apps.config.action_queue") +local channel = require("apps.config.channel") Leader = { config = { socket_file_name = {default='config-leader-socket'}, setup_fn = {required=true}, initial_configuration = {}, + follower_pids = {required=true}, Hz = {default=100}, - -- worker_app_name = {required=true} } } @@ -43,28 +45,43 @@ function Leader:new (conf) ret.current_app_graph = app_graph.new() ret.period = 1/conf.Hz ret.next_time = app.now() - ret:reset_configuration(conf.initial_configuration) + ret.followers = {} + for _,pid in ipairs(conf.follower_pids) do + table.insert(ret.followers, { pid=pid, queue={} }) + end ret.rpc_callee = rpc.prepare_callee('snabb-config-leader-v1') ret.rpc_handler = rpc.dispatch_handler(ret, 'rpc_') + + ret:reset_configuration(conf.initial_configuration) + return ret end function Leader:reset_configuration (configuration) - self:flush_config_action_queue() local new_app_graph = self.setup_fn(configuration) - self.config_action_queue = app.compute_config_actions(self.current_app_graph, - new_app_graph) + local actions = app.compute_config_actions(self.current_app_graph, + new_app_graph) + self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph self.current_configuration = configuration end -function Leader:flush_config_action_queue (configuration) - -- Once we switch to multiple processes, this will wait until all - -- followers have processed the actions. For now it just runs all - -- the actions. - if self.config_action_queue then - app.apply_config_actions(self.config_action_queue) - self.config_action_queue = nil +function Leader:take_follower_message_queue () + local actions = self.config_action_queue + self.config_action_queue = nil + return actions +end + +function Leader:enqueue_config_actions (actions) + local messages = {} + for _,action in ipairs(actions) do + local buf, len = action_queue.encode_action(action) + table.insert(messages, { buf=buf, len=len }) + end + for _,follower in ipairs(self.followers) do + for _,message in ipairs(messages) do + table.insert(follower.queue, message) + end end end @@ -76,10 +93,7 @@ function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end -function Leader:pull () - if app.now() < self.next_time then return end - self.next_time = app.now() + self.period - self:flush_config_action_queue() +function Leader:handle_calls_from_peers() local peers = self.peers while true do local sa = S.t.sockaddr_un() @@ -184,6 +198,36 @@ function Leader:pull () end end +function Leader:send_messages_to_followers() + for _,follower in ipairs(self.followers) do + if not follower.channel then + local name = '/'..tostring(follower.pid)..'/config-follower-channel' + -- local success, channel = pcall(channel.open, name) + --if success then follower.channel = channel end + follower.channel = channel.open(name) + end + local channel = follower.channel + if channel then + local queue = follower.queue + follower.queue = {} + local requeue = false + for _,msg in ipairs(queue) do + if not requeue then + requeue = not channel:put_message(msg.buf, msg.len) + end + if requeue then table.insert(follower.queue, msg) end + end + end + end +end + +function Leader:pull () + if app.now() < self.next_time then return end + self.next_time = app.now() + self.period + self:handle_calls_from_peers() + self:send_messages_to_followers() +end + function Leader:stop () for _,peer in ipairs(self.peers) do peer.fd:close() end self.peers = {} @@ -193,13 +237,29 @@ end function selftest () print('selftest: apps.config.leader') - local pcap = require("apps.pcap.pcap") - local Match = require("apps.test.match").Match - local c = config.new() - local function setup_fn(cfg) return app_graph.new() end - config.app(c, "leader", Leader, {setup_fn=setup_fn}) - engine.configure(c) - engine.main({ duration = 0.0001, report = {showapps=true,showlinks=true}}) - engine.configure(config.new()) + local graph = app_graph.new() + local function setup_fn(cfg) + local graph = app_graph.new() + local basic_apps = require('apps.basic.basic_apps') + app_graph.app(graph, "source", basic_apps.Source, {}) + app_graph.app(graph, "sink", basic_apps.Sink, {}) + app_graph.link(graph, "source.foo -> sink.bar") + return graph + end + app_graph.app(graph, "leader", Leader, + {setup_fn=setup_fn, follower_pids={S.getpid()}}) + app_graph.app(graph, "follower", require('apps.config.follower').Follower, + {}) + engine.configure(graph) + engine.main({ duration = 0.05, report = {showapps=true,showlinks=true}}) + assert(app.app_table.source) + assert(app.app_table.sink) + assert(app.link_table["source.foo -> sink.bar"]) + local link = app.link_table["source.foo -> sink.bar"] + local counter = require('core.counter') + assert(counter.read(link.stats.txbytes) > 0) + assert(counter.read(link.stats.txbytes) == counter.read(link.stats.rxbytes)) + assert(counter.read(link.stats.txdrop) == 0) + engine.configure(app_graph.new()) print('selftest: ok') end diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 537cece361..3c9e3dcf97 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -239,7 +239,7 @@ local function init_transmit_icmpv4_reply (rate_limiting) end end -LwAftr = {} +LwAftr = { yang_schema = 'snabb-softwire-v1' } function LwAftr:new(conf) if conf.debug then debug = true end diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 3f8b9b10dc..fb193edd4f 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -2,6 +2,7 @@ module(..., package.seeall) local config = require("core.config") local leader = require("apps.config.leader") +local follower = require("apps.config.follower") local Intel82599 = require("apps.intel.intel_app").Intel82599 local PcapFilter = require("apps.packet_filter.pcap_filter").PcapFilter local V4V6 = require("apps.lwaftr.V4V6").V4V6 @@ -454,5 +455,7 @@ function with_leader(f, graph, conf, ...) return graph end config.app(graph, 'leader', leader.Leader, - { setup_fn = setup_fn, initial_configuration = conf }) + { setup_fn = setup_fn, initial_configuration = conf, + follower_pids = { require('syscall').getpid() }}) + config.app(graph, "follower", follower.Follower, {}) end From 514784ada3a8d1c485abc986053f893e44cfb57b Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Tue, 15 Nov 2016 11:04:01 +0000 Subject: [PATCH 217/631] core.worker: Updated API & removed dead code Updated the comment describing the API. (Should be replaced by documentation in src/README.md.) Removed config.configure(name, c) which is a remnant of the past design when we supposed that app network configurations would be lingua franca for parents to configure their children. That is now out of scope. --- src/core/worker.lua | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index 9769eccfd0..17e2b28960 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -4,10 +4,9 @@ module(..., package.seeall) -- API: --- start(name, core) +-- start(name, luacode) -- stop(name) -- status() -> table of { name = } --- configure(name, config) local lib = require("core.lib") local shm = require("core.shm") @@ -72,18 +71,6 @@ function status () return status end --- Configure a worker process with a new app network. -function configure (name, c) - -- Ensure "configs" shm counter exists for child to poll - local child = children[name] - local child_path = "group/child/"..name - if not child.configs then - child.configs = shm.map(child_path.."/configs", {"counter"}, false, true) - end - config.save(shm.path(child_path.."/config"), c) - counter.add(child.configs, 1) -end - -------------------------------------------------------------- -- Worker (child) process code -------------------------------------------------------------- From cdc3881b6b383b274f1e6c4b64889a4533135c11 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 14:35:30 +0100 Subject: [PATCH 218/631] Serialize {} instead of nil * src/apps/config/action_queue.lua: Avoid trying to serialize "nil"; instead for apps that take no config, pass {}. --- src/apps/config/action_queue.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/config/action_queue.lua b/src/apps/config/action_queue.lua index d764d50af4..25a2c48b93 100644 --- a/src/apps/config/action_queue.lua +++ b/src/apps/config/action_queue.lua @@ -123,6 +123,7 @@ local function encoder() yang.compile_data_for_schema_by_name(class.yang_schema, arg, file_name) else + if arg == nil then arg = {} end binary.compile_ad_hoc_lua_data_to_file(file_name, arg) end self:string(file_name) From 16db816f9afa6f329e85ee3283ac255cebb54ff7 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 14:36:29 +0100 Subject: [PATCH 219/631] lwaftr apps load counters themselves lwcounter.init_counters is idempotent, so instead of passing counters into apps, apps should call lwcounter.init_counters themselves. --- src/apps/lwaftr/ipv4_apps.lua | 5 +++-- src/apps/lwaftr/ipv6_apps.lua | 5 +++-- src/apps/lwaftr/lwaftr.lua | 3 ++- src/program/lwaftr/setup.lua | 11 ++++------- src/program/snabbvmx/lwaftr/setup.lua | 12 ------------ 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/apps/lwaftr/ipv4_apps.lua b/src/apps/lwaftr/ipv4_apps.lua index 11411ce234..65b2ebb4af 100644 --- a/src/apps/lwaftr/ipv4_apps.lua +++ b/src/apps/lwaftr/ipv4_apps.lua @@ -6,6 +6,7 @@ local fragmentv4 = require("apps.lwaftr.fragmentv4") local fragv4_h = require("apps.lwaftr.fragmentv4_hardened") local lwutil = require("apps.lwaftr.lwutil") local icmp = require("apps.lwaftr.icmp") +local lwcounter = require("apps.lwaftr.lwcounter") local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") @@ -37,7 +38,7 @@ function Reassembler:new(conf) local max_ipv4_reassembly_packets = assert(conf.max_ipv4_reassembly_packets) local max_fragments_per_reassembly_packet = assert(conf.max_fragments_per_reassembly_packet) local o = { - counters = assert(conf.counters, "Counters not initialized"), + counters = lwcounter.init_counters(), ctab = fragv4_h.initialize_frag_table(max_ipv4_reassembly_packets, max_fragments_per_reassembly_packet), } @@ -85,7 +86,7 @@ end function Fragmenter:new(conf) local o = { - counters = assert(conf.counters, "Counters not initialized"), + counters = lwcounter.init_counters(), mtu = assert(conf.mtu), } return setmetatable(o, {__index=Fragmenter}) diff --git a/src/apps/lwaftr/ipv6_apps.lua b/src/apps/lwaftr/ipv6_apps.lua index 82d75cb4c0..e23b05dbed 100644 --- a/src/apps/lwaftr/ipv6_apps.lua +++ b/src/apps/lwaftr/ipv6_apps.lua @@ -6,6 +6,7 @@ local fragv6_h = require("apps.lwaftr.fragmentv6_hardened") local ndp = require("apps.lwaftr.ndp") local lwutil = require("apps.lwaftr.lwutil") local icmp = require("apps.lwaftr.icmp") +local lwcounter = require("apps.lwaftr.lwcounter") local ethernet = require("lib.protocol.ethernet") local ipv6 = require("lib.protocol.ipv6") @@ -42,7 +43,7 @@ function ReassembleV6:new(conf) local max_ipv6_reassembly_packets = conf.max_ipv6_reassembly_packets local max_fragments_per_reassembly_packet = conf.max_fragments_per_reassembly_packet local o = { - counters = assert(conf.counters, "Counters not initialized"), + counters = lwcounter.init_counters(), ctab = fragv6_h.initialize_frag_table(max_ipv6_reassembly_packets, max_fragments_per_reassembly_packet), } @@ -91,7 +92,7 @@ end function Fragmenter:new(conf) local o = { - counters = assert(conf.counters, "Counters not initialized"), + counters = lwcounter.init_counters(), mtu = assert(conf.mtu), } return setmetatable(o, {__index=Fragmenter}) diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 3c9e3dcf97..118d670456 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -4,6 +4,7 @@ local bt = require("apps.lwaftr.binding_table") local constants = require("apps.lwaftr.constants") local dump = require('apps.lwaftr.dump') local icmp = require("apps.lwaftr.icmp") +local lwcounter = require("apps.lwaftr.lwcounter") local lwdebug = require("apps.lwaftr.lwdebug") local lwheader = require("apps.lwaftr.lwheader") local lwutil = require("apps.lwaftr.lwutil") @@ -258,7 +259,7 @@ function LwAftr:new(conf) end o.control = channel.create('lwaftr/control', messages.lwaftr_message_t) - o.counters = assert(conf.counters, "Counters not initialized") + o.counters = lwcounter.init_counters() o.transmit_icmpv6_reply = init_transmit_icmpv6_reply( conf.internal_interface.error_rate_limiting) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index fb193edd4f..53a1649695 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -26,29 +26,26 @@ function lwaftr_app(c, conf) assert(type(conf) == 'table') local function append(t, elem) table.insert(t, elem) end local function prepend(t, elem) table.insert(t, 1, elem) end - conf.counters = lwcounter.init_counters() config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = conf.external_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - conf.external_interface.reassembly.max_fragments_per_packet, - counters = conf.counters }) + conf.external_interface.reassembly.max_fragments_per_packet }) config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { max_ipv6_reassembly_packets = conf.internal_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - conf.internal_interface.reassembly.max_fragments_per_packet, - counters = conf.counters }) + conf.internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "icmpechov4", ipv4_apps.ICMPEcho, { address = convert_ipv4(conf.external_interface.ip) }) config.app(c, "icmpechov6", ipv6_apps.ICMPEcho, { address = conf.internal_interface.ip }) config.app(c, 'lwaftr', lwaftr.LwAftr, conf) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, - { mtu=conf.external_interface.mtu, counters=conf.counters }) + { mtu=conf.external_interface.mtu }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, - { mtu=conf.internal_interface.mtu, counters=conf.counters }) + { mtu=conf.internal_interface.mtu }) config.app(c, "ndp", ipv6_apps.NDP, { src_ipv6 = conf.internal_interface.ip, src_eth = conf.internal_interface.mac, diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index a6f5459436..18aa48041f 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -126,8 +126,6 @@ function lwaftr_app(c, conf, lwconf, sock_path) assert(type(lwconf) == 'table') print(("Hairpinning: %s"):format(yesno(lwconf.internal_interface.hairpinning))) - local counters = lwcounter.init_counters() - local virt_id = "vm_" .. conf.interface.id local phy_id = "nic_" .. conf.interface.id @@ -170,14 +168,12 @@ function lwaftr_app(c, conf, lwconf, sock_path) if conf.ipv6_interface.fragmentation then local mtu = conf.ipv6_interface.mtu or lwconf.internal_interface.mtu config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { - counters = counters, max_ipv6_reassembly_packets = lwconf.internal_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = lwconf.internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, { - counters = counters, mtu = mtu, }) config.link(c, v6_output .. " -> reassemblerv6.input") @@ -207,14 +203,12 @@ function lwaftr_app(c, conf, lwconf, sock_path) if conf.ipv4_interface.fragmentation then local mtu = conf.ipv4_interface.mtu or lwconf.external_interface.mtu config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { - counters = counters, max_ipv4_reassembly_packets = lwconf.external_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = lwconf.external_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, { - counters = counters, mtu = mtu }) config.link(c, v4_output .. " -> reassemblerv4.input") @@ -251,7 +245,6 @@ function lwaftr_app(c, conf, lwconf, sock_path) config.link(c, "nh_fwd4.wire -> " .. v4_input) v4_input, v4_output = "nh_fwd4.vm", "nh_fwd4.vm" - lwconf.counters = counters config.app(c, "lwaftr", lwaftr.LwAftr, lwconf) config.link(c, "nh_fwd6.service -> lwaftr.v6") config.link(c, "lwaftr.v6 -> nh_fwd6.service") @@ -334,14 +327,12 @@ local function lwaftr_app_check (c, conf, lwconf, sources, sinks) if conf.ipv6_interface.fragmentation then local mtu = conf.ipv6_interface.mtu or lwconf.internal_interface.mtu config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { - counters = counters, max_ipv6_reassembly_packets = lwconf.internal_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = lwconf.internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, { - counters = counters, mtu = mtu, }) config.link(c, v6_src .. " -> reassemblerv6.input") @@ -366,14 +357,12 @@ local function lwaftr_app_check (c, conf, lwconf, sources, sinks) if conf.ipv4_interface.fragmentation then local mtu = conf.ipv4_interface.mtu or lwconf.external_interface.mtu config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { - counters = counters, max_ipv4_reassembly_packets = lwconf.external_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = lwconf.external_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, { - counters = counters, mtu = mtu }) config.link(c, v4_src .. " -> reassemblerv4.input") @@ -405,7 +394,6 @@ local function lwaftr_app_check (c, conf, lwconf, sources, sinks) config.link(c, v4_src.."-> nh_fwd4.wire") config.link(c, "nh_fwd4.wire -> "..v4_sink) - lwconf.counters = lwcounter.init_counters() config.app(c, "lwaftr", lwaftr.LwAftr, lwconf) config.link(c, "nh_fwd6.service -> lwaftr.v6") config.link(c, "lwaftr.v6 -> nh_fwd6.service") From 902dd3451fdbe44735f238e5f0424d4bc3a1e037 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 14:39:07 +0100 Subject: [PATCH 220/631] Binary serialization uses alignment of type, not value Oddly, alignof('struct {uint32 a; uint32 b;}') can be different from the alignof values of that type that are references (&). I have no idea what's going on. Given that we're copying out the data anyway, no big deal. --- src/apps/lwaftr/rangemap.lua | 3 ++- src/lib/ctable.lua | 8 +++++--- src/lib/yang/binary.lua | 11 ++++++----- src/lib/yang/stream.lua | 7 ++++--- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/apps/lwaftr/rangemap.lua b/src/apps/lwaftr/rangemap.lua index 9cd3255421..de322a29d5 100644 --- a/src/apps/lwaftr/rangemap.lua +++ b/src/apps/lwaftr/rangemap.lua @@ -162,7 +162,8 @@ struct { function RangeMap:save(stream) local entry_size = ffi.sizeof(self.entry_type) - stream:write_ptr(range_map_header_t(self.size, entry_size)) + stream:write_ptr(range_map_header_t(self.size, entry_size), + range_map_header_t) stream:write_array(self.entries, self.entry_type, self.size) end diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 095f450d26..fdb94656de 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -204,7 +204,8 @@ end function CTable:save(stream) stream:write_ptr(header_t(self.size, self.occupancy, self.max_displacement, - self.max_occupancy_rate, self.min_occupancy_rate)) + self.max_occupancy_rate, self.min_occupancy_rate), + header_t) stream:write_array(self.entries, self.entry_type, self.size + self.max_displacement + 1) @@ -615,8 +616,9 @@ function selftest() file:write(ffi.string(ptr, size)) end local stream = {} - function stream:write_ptr(ptr) - write(ptr, ffi.sizeof(ptr)) + function stream:write_ptr(ptr, type) + assert(ffi.sizeof(ptr) == ffi.sizeof(type)) + write(ptr, ffi.sizeof(type)) end function stream:write_array(ptr, type, count) write(ptr, ffi.sizeof(type) * count) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index b83aa85582..cec9d82714 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -88,7 +88,7 @@ local function value_emitter(ctype) local buf = ffi.typeof('$[1]', type)() local function emit(val, stream) buf[0] = val - stream:write_ptr(buf) + stream:write_ptr(buf, type) end value_emitters[ctype] = emit return emit @@ -120,7 +120,7 @@ local function data_emitter(production) return function(data, stream) stream:write_stringref('cdata') stream:write_stringref(production.ctype) - stream:write_ptr(data) + stream:write_ptr(data, ffi.typeof(production.ctype)) end else local emit_member = visitn(production.members) @@ -233,12 +233,13 @@ function data_compiler_from_grammar(emit_data, schema_name, schema_revision) local header = header_t( MAGIC, VERSION, source_mtime.sec, source_mtime.nsec, strtab:intern(schema_name), strtab:intern(schema_revision or '')) - stream:write_ptr(header) -- Write with empty data_len etc, fix it later. + -- Write with empty data_len etc, fix it later. + stream:write_ptr(header, header_t) header.data_start = stream.written local u32buf = ffi.new('uint32_t[1]') function stream:write_uint32(val) u32buf[0] = val - return self:write_ptr(u32buf) + return self:write_ptr(u32buf, 'uint32_t') end function stream:write_stringref(str) return self:write_uint32(strtab:intern(str)) @@ -248,7 +249,7 @@ function data_compiler_from_grammar(emit_data, schema_name, schema_revision) header.strtab_start, header.strtab_len = strtab:emit(stream) stream:rewind() -- Fix up header. - stream:write_ptr(header) + stream:write_ptr(header, header_t) stream:close_and_rename() end end diff --git a/src/lib/yang/stream.lua b/src/lib/yang/stream.lua index fd5b0f688b..c0b2380abc 100644 --- a/src/lib/yang/stream.lua +++ b/src/lib/yang/stream.lua @@ -32,9 +32,10 @@ function open_output_byte_stream(filename) to_write = to_write - written end end - function ret:write_ptr(ptr) - self:align(ffi.alignof(ptr)) - self:write(ptr, ffi.sizeof(ptr)) + function ret:write_ptr(ptr, type) + assert(ffi.sizeof(ptr) == ffi.sizeof(type)) + self:align(ffi.alignof(type)) + self:write(ptr, ffi.sizeof(type)) end function ret:rewind() fd:seek(0, 'set') From e81de3919c728732cc269aa476ec36bf2849dcf0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 14:47:29 +0100 Subject: [PATCH 221/631] Fix lstruct serialization The serialized field count should be the actual field count. --- src/lib/yang/binary.lua | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index cec9d82714..5e79006495 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -127,14 +127,19 @@ local function data_emitter(production) local normalize_id = data.normalize_id return function(data, stream) stream:write_stringref('lstruct') - stream:write_uint32(table_size(data)) + local out = {} for _,k in ipairs(member_names) do local id = normalize_id(k) if data[id] ~= nil then - stream:write_stringref(id) - emit_member[k](data[id], stream) + table.insert(out, {id, emit_member[k], data[id]}) end end + stream:write_uint32(#out) + for _,elt in ipairs(out) do + local id, emit, data = unpack(elt) + stream:write_stringref(id) + emit(data, stream) + end end end end From df3cfd4f2207d99e9a9b458d88a8e2339ae751da Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 14:52:50 +0100 Subject: [PATCH 222/631] Fix lltable loading --- src/lib/yang/binary.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 5e79006495..fc260f8e56 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -370,10 +370,10 @@ local function read_compiled_data(stream, strtab) return cltable.build(keys, values) end function readers.lltable() - local ret = data.make_assoc() + local ret = {} for i=1,stream:read_uint32() do local k = read1() - ret:add(k, read1()) + ret[k] = read1() end return ret end From f8380e99d62b3c3410dc724f378c84f9d0e81585 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 14:51:42 +0100 Subject: [PATCH 223/631] Improve ad hoc compilation for legacy IPv4 addresses Legacy IPv4 addresses are represented as uint8_t[4], not uint32. Special-case them, and indeed all types that we care about. This code will go away anyway some day. --- src/lib/yang/binary.lua | 32 ++++++++++++++++---------------- src/lib/yang/value.lua | 6 ++++++ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index fc260f8e56..955053b287 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -289,22 +289,22 @@ local function ad_hoc_grammar_from_data(data) elseif type(data) == 'cdata' then -- Hackety hack. local ctype = tostring(ffi.typeof(data)):match('^ctype<(.*)>$') - if ctype:match('%*') then - error('pointer in ffi cdata cannot be serialized: '..ctype) - elseif ctype:match('%[') or ctype:match('^struct ') then - return {type='struct', members={}, ctype=ctype} - elseif ctype == 'uint64_t' then - return {type='scalar', argument_type={primitive_type='uint64'}} - elseif ctype == 'int64_t' then - return {type='scalar', argument_type={primitive_type='int64'}} - elseif ctype == 'double' or ctype == 'float' then - return {type='scalar', argument_type={primitive_type='decimal64'}} - elseif pcall(tonumber, data) then - local primitive_type = assert(ctype:match('^(.*)_t$')) - return {type='scalar', argument_type={primitive_type=primitive_type}} - else - error('unhandled ffi ctype: '..ctype) - end + local primitive_types = { + ['unsigned char [4]'] = 'legacy-ipv4-address', + ['unsigned char (&)[4]'] = 'legacy-ipv4-address', + ['unsigned char [6]'] = 'mac-address', + ['unsigned char (&)[6]'] = 'mac-address', + ['unsigned char [16]'] = 'ipv6-address', + ['unsigned char (&)[16]'] = 'ipv6-address', + ['uint8_t'] = 'uint8', ['int8_t'] = 'int8', + ['uint16_t'] = 'uint16', ['int16_t'] = 'int16', + ['uint32_t'] = 'uint32', ['int32_t'] = 'int32', + ['uint64_t'] = 'uint64', ['int64_t'] = 'int64', + ['double'] = 'decimal64' -- ['float'] = 'decimal64', + } + local prim = primitive_types[ctype] + if not prim then error('unhandled ffi ctype: '..ctype) end + return {type='scalar', argument_type={primitive_type=prim}} elseif type(data) == 'number' then return {type='scalar', argument_type={primitive_type='decimal64'}} elseif type(data) == 'string' then diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua index 4ffe1c30f2..3027907a74 100644 --- a/src/lib/yang/value.lua +++ b/src/lib/yang/value.lua @@ -107,6 +107,12 @@ types['ipv4-address'] = { tostring = function(val) return util.ipv4_ntop(val) end } +types['legacy-ipv4-address'] = { + ctype = 'uint8_t[4]', + parse = function(str, what) return assert(ipv4:pton(str)) end, + tostring = function(val) return ipv4:ntop(val) end +} + types['ipv6-address'] = { ctype = 'uint8_t[16]', parse = function(str, what) return assert(ipv6:pton(str)) end, From 8866ee2d13334c1fd711a664b846e8b25c10442f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 15:10:37 +0100 Subject: [PATCH 224/631] Avoid too much type generation in yang module Every time you ffi.typeof an anonymous struct, it makes a new type. Avoid that by caching all types of interest. --- src/lib/yang/binary.lua | 12 +++++++----- src/lib/yang/data.lua | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 955053b287..26df481745 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -82,7 +82,7 @@ end local value_emitters = {} local function value_emitter(ctype) if value_emitters[ctype] then return value_emitters[ctype] end - local type = ffi.typeof(ctype) + local type = data.typeof(ctype) local align = ffi.alignof(type) local size = ffi.sizeof(type) local buf = ffi.typeof('$[1]', type)() @@ -117,10 +117,11 @@ local function data_emitter(production) for k,_ in pairs(production.members) do table.insert(member_names, k) end table.sort(member_names) if production.ctype then + local typeof = data.typeof return function(data, stream) stream:write_stringref('cdata') stream:write_stringref(production.ctype) - stream:write_ptr(data, ffi.typeof(production.ctype)) + stream:write_ptr(data, typeof(production.ctype)) end else local emit_member = visitn(production.members) @@ -145,11 +146,12 @@ local function data_emitter(production) end function handlers.array(production) if production.ctype then + local typeof = data.typeof return function(data, stream) stream:write_stringref('carray') stream:write_stringref(production.ctype) stream:write_uint32(#data) - stream:write_array(data.ptr, ffi.typeof(production.ctype), #data) + stream:write_array(data.ptr, typeof(production.ctype), #data) end else local emit_tagged_value = visit1( @@ -330,7 +332,7 @@ local function read_compiled_data(stream, strtab) end local ctypes = {} local function scalar_type(ctype) - if not ctypes[ctype] then ctypes[ctype] = ffi.typeof(ctype) end + if not ctypes[ctype] then ctypes[ctype] = data.typeof(ctype) end return ctypes[ctype] end @@ -360,7 +362,7 @@ local function read_compiled_data(stream, strtab) function readers.ctable() local key_ctype = read_string() local value_ctype = read_string() - local key_t, value_t = ffi.typeof(key_ctype), ffi.typeof(value_ctype) + local key_t, value_t = data.typeof(key_ctype), data.typeof(value_ctype) return ctable.load(stream, {key_type=key_t, value_type=value_t}) end function readers.cltable() diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index b4e317b89e..70f1ddcf93 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -14,6 +14,15 @@ function normalize_id(id) return id:gsub('[^%w_]', '_') end +-- Avoid generating lots of struct types. Note that this function is +-- only for string type names without parameters. +local type_cache = {} +function typeof(name) + assert(type(name) == 'string') + if not type_cache[name] then type_cache[name] = ffi.typeof(name) end + return type_cache[name] +end + -- If a "list" node has one key that is string-valued, we will represent -- instances of that node as normal Lua tables where the key is the -- table key and the value does not contain the key. @@ -187,7 +196,7 @@ local function struct_parser(keyword, members, ctype) assert_not_duplicate(out, keyword) return parse1(node) end - local struct_t = ctype and ffi.typeof(ctype) + local struct_t = ctype and typeof(ctype) local function finish(out) -- FIXME check mandatory values. if struct_t then return struct_t(out) else return out end @@ -206,7 +215,7 @@ local function array_parser(keyword, element_type, ctype) table.insert(out, parse1(node)) return out end - local elt_t = ctype and ffi.typeof(ctype) + local elt_t = ctype and typeof(ctype) local array_t = ctype and ffi.typeof('$[?]', elt_t) local function finish(out) -- FIXME check min-elements @@ -299,8 +308,8 @@ local function table_parser(keyword, keys, values, string_key, key_ctype, for k,v in pairs(keys) do members[k] = v end for k,v in pairs(values) do members[k] = v end local parser = struct_parser(keyword, members) - local key_t = key_ctype and ffi.typeof(key_ctype) - local value_t = value_ctype and ffi.typeof(value_ctype) + local key_t = key_ctype and typeof(key_ctype) + local value_t = value_ctype and typeof(value_ctype) local init if key_t and value_t then function init() return ctable_builder(key_t, value_t) end From 53c2e484ae60005523b9c86e5084596db41e9da9 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 15:17:04 +0100 Subject: [PATCH 225/631] Follower flushes JIT after starting or reconfiguring apps Also remove a print. --- src/apps/config/action_queue.lua | 1 - src/apps/config/follower.lua | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/apps/config/action_queue.lua b/src/apps/config/action_queue.lua index 25a2c48b93..92f50118b5 100644 --- a/src/apps/config/action_queue.lua +++ b/src/apps/config/action_queue.lua @@ -113,7 +113,6 @@ local function encoder() end function encoder:class(class) local require_path, name = find_public_name(class) - print('encoding', require_path, name) self:string(require_path) self:string(name) end diff --git a/src/apps/config/follower.lua b/src/apps/config/follower.lua index 28c438013b..4876d0b9a9 100644 --- a/src/apps/config/follower.lua +++ b/src/apps/config/follower.lua @@ -28,13 +28,18 @@ end function Follower:handle_actions_from_leader() local channel = self.channel + local should_flush = false while true do local buf, len = channel:peek_message() if not buf then break end local action = action_queue.decode_action(buf, len) app.apply_config_actions({action}) channel:discard_message(len) + if action[1] == 'start_app' or action[1] == 'reconfig_app' then + should_flush = true + end end + if should_flush then require('jit').flush() end end function Follower:pull () From 3f6644d6408dcc54c724f392a51a7e9a619dcbc1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 15:53:23 +0100 Subject: [PATCH 226/631] Make get-config work --- src/apps/config/leader.lua | 20 ++++++++++++++++---- src/program/lwaftr/setup.lua | 3 ++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 06883d40a2..f466a78a1e 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -5,6 +5,7 @@ module(...,package.seeall) local S = require("syscall") local ffi = require("ffi") local yang = require("lib.yang.yang") +local data = require("lib.yang.data") local rpc = require("lib.yang.rpc") local app = require("core.app") local shm = require("core.shm") @@ -16,7 +17,9 @@ Leader = { config = { socket_file_name = {default='config-leader-socket'}, setup_fn = {required=true}, - initial_configuration = {}, + -- Could relax this requirement. + initial_configuration = {required=true}, + schema_name = {required=true}, follower_pids = {required=true}, Hz = {default=100}, } @@ -39,6 +42,7 @@ function Leader:new (conf) local instance_dir = shm.root..'/'..tostring(S.getpid()) ret.socket_file_name = instance_dir..'/'..ret.socket_file_name end + ret.schema_name = conf.schema_name ret.socket = open_socket(ret.socket_file_name) ret.peers = {} ret.setup_fn = conf.setup_fn @@ -85,8 +89,13 @@ function Leader:enqueue_config_actions (actions) end end -function Leader:rpc_get_config (data) - return { config = "hey!" } +function Leader:rpc_get_config (args) + -- FIXME: Push more of this to a lib. + local schema = yang.load_schema_by_name(self.schema_name) + local grammar = data.data_grammar_from_schema(schema) + local printer = data.data_string_printer_from_grammar(grammar) + local config_str = printer(self.current_configuration) + return { config = config_str } end function Leader:handle (payload) @@ -247,7 +256,10 @@ function selftest () return graph end app_graph.app(graph, "leader", Leader, - {setup_fn=setup_fn, follower_pids={S.getpid()}}) + {setup_fn=setup_fn, follower_pids={S.getpid()}, + -- Use a schema with no data nodes, just for + -- testing. + schema_name='ietf-inet-types', initial_configuration={}}) app_graph.app(graph, "follower", require('apps.config.follower').Follower, {}) engine.configure(graph) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 53a1649695..85cde93f08 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -453,6 +453,7 @@ function with_leader(f, graph, conf, ...) end config.app(graph, 'leader', leader.Leader, { setup_fn = setup_fn, initial_configuration = conf, - follower_pids = { require('syscall').getpid() }}) + follower_pids = { require('syscall').getpid() }, + schema_name = 'snabb-softwire-v1'}) config.app(graph, "follower", follower.Follower, {}) end From 2a064891db8d7a5b2f42dc2348265a57bf08b40e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 16:04:41 +0100 Subject: [PATCH 227/631] YANG parser adds location information to result --- src/lib/yang/parser.lua | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 7852513ef7..e44b33f216 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -38,11 +38,15 @@ function Parser.new(str, filename) return ret end +function Parser:loc() + return string.format('%s:%d:%d', self.name or '', self.line, + self.column) +end + function Parser:error(msg, ...) print(self.str:match("[^\n]*", self.line_pos)) print(string.rep(" ", self.column).."^") - error(('%s:%d:%d: error: '..msg):format( - self.name or '', self.line, self.column, ...)) + error(('%s: error: '..msg):format(self:loc(), ...)) end function Parser:read_char() @@ -237,7 +241,7 @@ end function Parser:parse_statement() self:skip_whitespace() - local returnval = {} + local returnval = { loc = self:loc() } -- Then must be a string that is the statement's identifier local keyword = self:parse_keyword() @@ -289,10 +293,19 @@ function selftest() end end + local function strip_locs(exp) + if type(exp) ~= 'table' then return exp end + local ret = {} + for k, v in pairs(exp) do + if k ~= 'loc' then ret[k] = strip_locs(v) end + end + return ret + end + local function test_string(src, exp) local parser = Parser.new(src) parser:skip_whitespace() - assert_equal(parser:parse_string(), exp) + assert_equal(strip_locs(parser:parse_string()), exp) end local function pp(x) @@ -317,7 +330,7 @@ function selftest() local function test_module(src, exp) - local result = parse_string(src) + local result = strip_locs(parse_string(src)) if not lib.equal(result, exp) then pp(result) pp(exp) From eaeb65da6222de1fda6c7217f6a5e6214e32e6cb Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Tue, 15 Nov 2016 16:26:55 +0100 Subject: [PATCH 228/631] Add CSV output to the lwaftr loadtest command (#578) --- src/program/lwaftr/loadtest/README | 14 +++-- src/program/lwaftr/loadtest/loadtest.lua | 59 ++++++++++++++++------ src/program/lwaftr/loadtest/promise.lua | 2 + src/program/lwaftr/transient/transient.lua | 2 +- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/program/lwaftr/loadtest/README b/src/program/lwaftr/loadtest/README index 968a1ad6f9..5c731183b8 100644 --- a/src/program/lwaftr/loadtest/README +++ b/src/program/lwaftr/loadtest/README @@ -8,6 +8,11 @@ Usage: loadtest [OPTIONS] [ 0, 'bitrate must be positive') assert(opts.step > 0, 'step must be positive') @@ -198,50 +204,72 @@ function run(args) ret[stream.nic_tx_id] = { tx = read_counters(tx_nic.input.rx), rx = read_counters(rx_nic.output.tx), - drop = rx_nic:ingress_packet_drops() + drop = rx_nic:ingress_packet_drops() } end return ret end - function tester.print_counter_diff(before, after, duration) + function tester.print_counter_diff( + before, after, duration, bench_file, gbps_bitrate) local function bitrate(diff) -- 7 bytes preamble, 1 start-of-frame, 4 CRC, 12 interpacket gap. local overhead = 7 + 1 + 4 + 12 return (diff.txbytes + diff.txpackets * overhead) * 8 / duration end for _, stream in ipairs(streams) do + bench_file:write(('%f,%s'):format(gbps_bitrate, stream.tx_name)) print(string.format(' %s:', stream.tx_name)) local nic_id = stream.nic_tx_id local nic_before, nic_after = before[nic_id], after[nic_id] local tx = diff_counters(nic_before.tx, nic_after.tx) + local tx_mpps = tx.txpackets / duration / 1e6 + local tx_gbps = bitrate(tx) / 1e9 local rx = diff_counters(nic_before.rx, nic_after.rx) + local rx_mpps = rx.txpackets / duration / 1e6 + local rx_gbps = bitrate(rx) / 1e9 local drop = tonumber(nic_after.drop - nic_before.drop) + local lost_packets = (tx.txpackets - rx.txpackets) - drop + local lost_percent = (tx.txpackets - rx.txpackets) / tx.txpackets * 100 print(string.format(' TX %d packets (%f MPPS), %d bytes (%f Gbps)', - tx.txpackets, tx.txpackets / duration / 1e6, - tx.txbytes, bitrate(tx) / 1e9)) + tx.txpackets, tx_mpps, tx.txbytes, tx_gbps)) + bench_file:write((',%d,%f,%d,%f'):format( + tx.txpackets, tx_mpps, tx.txbytes, tx_gbps)) print(string.format(' RX %d packets (%f MPPS), %d bytes (%f Gbps)', - rx.txpackets, rx.txpackets / duration / 1e6, - rx.txbytes, bitrate(rx) / 1e9)) + rx.txpackets, rx_mpps, rx.txbytes, rx_gbps)) + bench_file:write((',%d,%f,%d,%f'):format( + rx.txpackets, rx_mpps, rx.txbytes, rx_gbps)) print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', - drop, (tx.txpackets - rx.txpackets) - drop, - (tx.txpackets - rx.txpackets) / tx.txpackets * 100)) + drop, lost_packets, lost_percent)) + bench_file:write((',%d,%d,%f\n'):format( + drop, lost_packets, lost_percent)) end + bench_file:flush() end - function tester.measure(bitrate, duration) + function tester.measure(bitrate, duration, bench_file) + local gbps_bitrate = bitrate/1e9 local start_counters = tester.record_counters() local function report() local end_counters = tester.record_counters() - tester.print_counter_diff(start_counters, end_counters, duration) + tester.print_counter_diff( + start_counters, end_counters, duration, bench_file, gbps_bitrate) end - print(string.format('Applying %f Gbps of load.', bitrate/1e9)) + print(string.format('Applying %f Gbps of load.', gbps_bitrate)) return tester.generate_load(bitrate, duration): -- Wait 2ms for packets in flight to arrive and_then(promise.Wait, 0.002): and_then(report) end + local function create_bench_file(filename) + local bench_file = io.open(filename, "w") + bench_file:write("load_gbps,stream,tx_packets,tx_mpps,tx_bytes,tx_gbps".. + ",rx_packets,rx_mpps,rx_bytes,rx_gbps,ingress_drop,lost_packets,lost_percent\n") + bench_file:flush() + return bench_file + end + local function run_engine(head, tail) local is_done = false local function mark_done() is_done = true end @@ -252,6 +280,7 @@ function run(args) engine.main({done=done}) end + opts.bench_file = create_bench_file(opts.bench_file) engine.busywait = true local head = promise.new() run_engine(head, diff --git a/src/program/lwaftr/loadtest/promise.lua b/src/program/lwaftr/loadtest/promise.lua index 1f28adb247..9b8af97663 100644 --- a/src/program/lwaftr/loadtest/promise.lua +++ b/src/program/lwaftr/loadtest/promise.lua @@ -12,6 +12,8 @@ local function curry(f, ...) if #curried_args == 0 then return f end return function(...) local args = { ... } + -- Prepend the curried args to the passed ones, in the correct order: + -- if curried_args == (A, B), args == (C, D) -> (B, C, D) -> (A, B, C, D) for i=#curried_args, 1, -1 do table.insert(args, 1, curried_args[i]) end diff --git a/src/program/lwaftr/transient/transient.lua b/src/program/lwaftr/transient/transient.lua index 00110a3f64..727d27c83e 100644 --- a/src/program/lwaftr/transient/transient.lua +++ b/src/program/lwaftr/transient/transient.lua @@ -64,7 +64,7 @@ function parse_args(args) function handlers.h() show_usage(0) end args = lib.dogetopt(args, handlers, "hb:s:D:p:", { bitrate="b", step="s", duration="D", period="p", - ["bench-file"]=0, help="h" }) + ["bench-file"]=1, help="h" }) if not opts.step then opts.step = opts.bitrate / 10 end assert(opts.bitrate > 0, 'bitrate must be positive') assert(opts.step > 0, 'step must be positive') From d651693dd073df78ded75afdf7420989cf18c504 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 16:28:33 +0100 Subject: [PATCH 229/631] Schema strips annotations earlier --- src/lib/yang/schema.lua | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 14b83293bf..10b285a1fe 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -33,6 +33,8 @@ local function parse_node(src, parent_path, order) ret = setmetatable(ret, {__index=Node}) local initialize = initializers[ret.kind] if initialize then initialize(ret) end + -- Strip annotations. + ret.path, ret.order, ret.argument_string, ret.children = nil return ret end @@ -561,18 +563,6 @@ local function schema_from_ast(ast) return ret end --- Strip properties pertaining to original source representation. -local function strip(exp) - if type(exp) ~= 'table' then return exp end - local ret = {} - for k, v in pairs(exp) do - if k ~= 'children' and k ~= 'argument_string' and k ~= 'order' and k ~= 'path' then - ret[k] = strip(v) - end - end - return ret -end - local function set(...) local ret = {} for k, v in pairs({...}) do ret[v] = true end @@ -815,10 +805,10 @@ function parse_schema_file(filename) end function load_schema(src, filename) - return resolve(primitivize(strip(parse_schema(src, filename)))) + return resolve(primitivize(parse_schema(src, filename))) end function load_schema_file(filename) - return resolve(primitivize(strip(parse_schema_file(filename)))) + return resolve(primitivize(parse_schema_file(filename))) end function load_schema_by_name(name, revision) -- FIXME: @ is not valid in a Lua module name. From a43c4d55502beb3fd4e123bcfb47bdad0c67feb3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 16:36:37 +0100 Subject: [PATCH 230/631] Schema parsing moving away from mutation Instead we build up results instead of mutating them. Eventually this will stop annotating the node with the path, children, and all that. --- src/lib/yang/schema.lua | 155 ++++++++++++++++++++-------------------- 1 file changed, 79 insertions(+), 76 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 10b285a1fe..7d0c774094 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -29,10 +29,11 @@ local function parse_node(src, parent_path, order) end ret.order = order ret.argument_string = src.argument - ret.children = parse_children(src, ret.path) + local children = parse_children(src, ret.path) + ret.children = children ret = setmetatable(ret, {__index=Node}) local initialize = initializers[ret.kind] - if initialize then initialize(ret) end + if initialize then initialize(ret, src, children) end -- Strip annotations. ret.path, ret.order, ret.argument_string, ret.children = nil return ret @@ -69,12 +70,14 @@ local function parse_range(node, range) else return res end end -local function collect_children(node, kinds) - if type(kinds) == 'string' then return collect_children(node, {kinds}) end +local function collect_children(children, kinds) + if type(kinds) == 'string' then + return collect_children(children, {kinds}) + end local ret = {} for _, kind in ipairs(kinds) do - if node.children[kind] then - for _, child in pairs(node.children[kind]) do + if children[kind] then + for _, child in pairs(children[kind]) do table.insert(ret, child) end end @@ -84,7 +87,7 @@ end local function collect_children_by_prop(node, kinds, prop) local ret = {} - for _, child in ipairs(collect_children(node, kinds)) do + for _, child in ipairs(collect_children(node.children, kinds)) do assert_with_path(child[prop], node.path, 'child of kind %s missing prop %s', child.kind, prop) assert_with_path(not ret[child[prop]], node.path, @@ -128,16 +131,16 @@ local function collect_data_or_case_children_at_least_1(node) return ret end -local function collect_child_properties(node, kind, field) +local function collect_child_properties(children, kind, field) local ret = {} - for _, child in ipairs(collect_children(node, kind)) do + for _, child in ipairs(collect_children(children, kind)) do table.insert(ret, child[field]) end return ret end local function maybe_child(node, kind) - local children = collect_children(node, kind) + local children = collect_children(node.children, kind) if #children > 1 then error_with_path(node.path, 'expected at most one child of type %s', kind) end @@ -162,17 +165,17 @@ end -- Simple statement kinds with string, natural, or boolean values all -- just initialize by parsing their argument and storing it as the -- "value" property in the schema node. -local function init_string(node) +local function init_string(node, src, children) node.value = require_argument(node) end -local function init_natural(node) +local function init_natural(node, src, children) local arg = require_argument(node) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, node.path, 'not a natural number: %s', arg) node.value = as_num end -local function init_boolean(node) +local function init_boolean(node, src, children) local arg = require_argument(node) if arg == 'true' then node.value = true elseif arg == 'false' then node.value = false @@ -182,47 +185,47 @@ end -- For all other statement kinds, we have custom initializers that -- parse out relevant sub-components and store them as named -- properties on the schema node. -local function init_anyxml(node) +local function init_anyxml(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.must = collect_child_properties(node, 'must', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') + node.must = collect_child_properties(children, 'must', 'value') node.config = maybe_child_property(node, 'config', 'value') node.mandatory = maybe_child_property(node, 'mandatory', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_argument(node) +local function init_argument(node, src, children) node.id = require_argument(node) node.yin_element = maybe_child_property(node, 'yin-element', 'value') end -local function init_augment(node) +local function init_augment(node, src, children) node.node_id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') node.body = collect_data_or_case_children_at_least_1(node) end -local function init_belongs_to(node) +local function init_belongs_to(node, src, children) node.id = require_argument(node) node.prefix = require_child(node, 'prefix').value end -local function init_case(node) +local function init_case(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') node.body = collect_body_children(node) end -local function init_choice(node) +local function init_choice(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.default = maybe_child_property(node, 'default', 'value') node.config = maybe_child_property(node, 'config', 'value') node.mandatory = maybe_child_property(node, 'mandatory', 'value') @@ -235,11 +238,11 @@ local function init_choice(node) node, {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) end -local function init_container(node) +local function init_container(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.must = collect_child_properties(node, 'must', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') + node.must = collect_child_properties(children, 'must', 'value') node.presence = maybe_child_property(node, 'presence', 'value') node.config = maybe_child_property(node, 'config', 'value') node.status = maybe_child_property(node, 'status', 'value') @@ -249,21 +252,21 @@ local function init_container(node) node.groupings = collect_children_by_id(node, 'grouping') node.body = collect_body_children(node) end -local function init_extension(node) +local function init_extension(node, src, children) node.id = require_argument(node) node.argument = maybe_child_property(node, 'argument', 'id') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_feature(node) +local function init_feature(node, src, children) node.id = require_argument(node) - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_grouping(node) +local function init_grouping(node, src, children) node.id = require_argument(node) node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') @@ -272,34 +275,34 @@ local function init_grouping(node) node.groupings = collect_children_by_id(node, 'grouping') node.body = collect_body_children(node) end -local function init_identity(node) +local function init_identity(node, src, children) node.id = require_argument(node) node.base = maybe_child_property(node, 'base', 'id') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_import(node) +local function init_import(node, src, children) node.id = require_argument(node) node.prefix = require_child_property(node, 'prefix', 'value') node.revision_date = maybe_child_property(node, 'revision-date', 'value') end -local function init_include(node) +local function init_include(node, src, children) node.id = require_argument(node) node.revision_date = maybe_child_property(node, 'revision-date', 'value') end -local function init_input(node) +local function init_input(node, src, children) node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') node.body = collect_body_children_at_least_1(node) end -local function init_leaf(node) +local function init_leaf(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.type = require_child(node, 'type') node.units = maybe_child_property(node, 'units', 'value') - node.must = collect_child_properties(node, 'must', 'value') + node.must = collect_child_properties(children, 'must', 'value') node.default = maybe_child_property(node, 'default', 'value') node.config = maybe_child_property(node, 'config', 'value') node.mandatory = maybe_child_property(node, 'mandatory', 'value') @@ -307,13 +310,13 @@ local function init_leaf(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_leaf_list(node) +local function init_leaf_list(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.type = require_child(node, 'type') node.units = maybe_child_property(node, 'units', 'value') - node.must = collect_child_properties(node, 'must', 'value') + node.must = collect_child_properties(children, 'must', 'value') node.config = maybe_child_property(node, 'config', 'value') node.min_elements = maybe_child_property(node, 'min-elements', 'value') node.max_elements = maybe_child_property(node, 'max-elements', 'value') @@ -322,19 +325,19 @@ local function init_leaf_list(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_length(node) +local function init_length(node, src, children) -- TODO: parse length arg str node.value = require_argument(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_list(node) +local function init_list(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') - node.must = collect_child_properties(node, 'must', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') + node.must = collect_child_properties(children, 'must', 'value') node.key = maybe_child_property(node, 'key', 'value') - node.unique = collect_child_properties(node, 'unique', 'value') + node.unique = collect_child_properties(children, 'unique', 'value') node.config = maybe_child_property(node, 'config', 'value') node.min_elements = maybe_child_property(node, 'min-elements', 'value') node.max_elements = maybe_child_property(node, 'max-elements', 'value') @@ -346,7 +349,7 @@ local function init_list(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_module(node) +local function init_module(node, src, children) node.id = require_argument(node) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.namespace = require_child_property(node, 'namespace', 'value') @@ -357,8 +360,8 @@ local function init_module(node) node.contact = maybe_child_property(node, 'contact', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.revisions = collect_children(node, 'revision') - node.augments = collect_children(node, 'augment') + node.revisions = collect_children(children, 'revision') + node.augments = collect_children(children, 'augment') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') node.features = collect_children_by_id(node, 'feature') @@ -369,13 +372,13 @@ local function init_module(node) node.deviations = collect_children_by_id(node, 'deviation') node.body = collect_body_children(node) end -local function init_namespace(node) +local function init_namespace(node, src, children) -- TODO: parse uri? node.value = require_argument(node) end -local function init_notification(node) +local function init_notification(node, src, children) node.id = require_argument(node) - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -383,29 +386,29 @@ local function init_notification(node) node.groupings = collect_children_by_id(node, 'grouping') node.body = collect_body_children(node) end -local function init_output(node) +local function init_output(node, src, children) node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') node.body = collect_body_children_at_least_1(node) end -local function init_path(node) +local function init_path(node, src, children) -- TODO: parse path string node.value = require_argument(node) end -local function init_pattern(node) +local function init_pattern(node, src, children) node.value = require_argument(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_range(node) +local function init_range(node, src, children) node.value = parse_range(node, require_argument(node)) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_refine(node) +local function init_refine(node, src, children) node.node_id = require_argument(node) -- All subnode kinds. - node.must = collect_child_properties(node, 'must', 'value') + node.must = collect_child_properties(children, 'must', 'value') node.config = maybe_child_property(node, 'config', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -418,15 +421,15 @@ local function init_refine(node) node.min_elements = maybe_child_property(node, 'min-elements', 'value') node.max_elements = maybe_child_property(node, 'max-elements', 'value') end -local function init_revision(node) +local function init_revision(node, src, children) -- TODO: parse date node.value = require_argument(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_rpc(node) +local function init_rpc(node, src, children) node.id = require_argument(node) - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -435,21 +438,21 @@ local function init_rpc(node) node.input = maybe_child(node, 'input') node.output = maybe_child(node, 'output') end -local function init_type(node) +local function init_type(node, src, children) node.id = require_argument(node) node.range = maybe_child(node, 'range') node.fraction_digits = maybe_child_property(node, 'fraction-digits', 'value') node.length = maybe_child_property(node, 'length', 'value') - node.patterns = collect_children(node, 'pattern') - node.enums = collect_children(node, 'enum') + node.patterns = collect_children(children, 'pattern') + node.enums = collect_children(children, 'enum') -- !!! path node.leafref = maybe_child_property(node, 'path', 'value') - node.require_instances = collect_children(node, 'require-instance') + node.require_instances = collect_children(children, 'require-instance') node.identityref = maybe_child_property(node, 'base', 'value') - node.union = collect_children(node, 'type') - node.bits = collect_children(node, 'bit') + node.union = collect_children(children, 'type') + node.bits = collect_children(children, 'bit') end -local function init_submodule(node) +local function init_submodule(node, src, children) node.id = require_argument(node) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.belongs_to = require_child(node, 'belongs-to') @@ -459,8 +462,8 @@ local function init_submodule(node) node.contact = maybe_child_property(node, 'contact', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.revisions = collect_children(node, 'revision') - node.augments = collect_children(node, 'augment') + node.revisions = collect_children(children, 'revision') + node.augments = collect_children(children, 'augment') node.typedefs = collect_children_by_id(node, 'typedef') node.groupings = collect_children_by_id(node, 'grouping') node.features = collect_children_by_id(node, 'feature') @@ -471,7 +474,7 @@ local function init_submodule(node) node.deviations = collect_children_by_id(node, 'deviation') node.body = collect_body_children(node) end -local function init_typedef(node) +local function init_typedef(node, src, children) node.id = require_argument(node) node.type = require_child(node, 'type') node.units = maybe_child_property(node, 'units', 'value') @@ -480,18 +483,18 @@ local function init_typedef(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_uses(node) +local function init_uses(node, src, children) node.id = require_argument(node) node.when = maybe_child_property(node, 'when', 'value') - node.if_features = collect_child_properties(node, 'if-feature', 'value') + node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') node.typedefs = collect_children_by_id(node, 'typedef') - node.refines = collect_children(node, 'refine') - node.augments = collect_children(node, 'augment') + node.refines = collect_children(children, 'refine') + node.augments = collect_children(children, 'augment') end -local function init_value(node) +local function init_value(node, src, children) local arg = require_argument(node) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num, From b81c4959d4c200d02eed85ca12a681c4cd1d37e6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 16:38:32 +0100 Subject: [PATCH 231/631] Config leader checks path Before, the path attribute was getting stripped out of the schema, due to heavihandedness. --- src/apps/config/leader.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index f466a78a1e..3ff6d48b7a 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -91,6 +91,7 @@ end function Leader:rpc_get_config (args) -- FIXME: Push more of this to a lib. + assert(args.path == '/') local schema = yang.load_schema_by_name(self.schema_name) local grammar = data.data_grammar_from_schema(schema) local printer = data.data_string_printer_from_grammar(grammar) From fbebbb91ff5dbc05ae6c92ba721d814ce52a35fc Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 17:05:36 +0100 Subject: [PATCH 232/631] Minor schema refactor --- src/lib/yang/schema.lua | 80 ++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 7d0c774094..e68f23785c 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -4,6 +4,13 @@ module(..., package.seeall) local parser = require("lib.yang.parser") local util = require("lib.yang.util") +local function error_with_loc(loc, msg, ...) + error(string.format("%s: "..msg, loc, ...)) +end +local function assert_with_loc(expr, loc, msg, ...) + if not expr then error_with_path(loc, msg, ...) end + return expr +end local function error_with_path(path, msg, ...) error(string.format("%s: "..msg, path, ...)) end @@ -49,9 +56,8 @@ function parse_children(src, parent_path) return ret end -local function require_argument(node) - return assert_with_path(node.argument_string, node.path, - 'missing argument') +local function require_argument(src) + return assert_with_loc(src.argument, src.loc, 'missing argument') end local function parse_range(node, range) @@ -166,17 +172,17 @@ end -- just initialize by parsing their argument and storing it as the -- "value" property in the schema node. local function init_string(node, src, children) - node.value = require_argument(node) + node.value = require_argument(src) end local function init_natural(node, src, children) - local arg = require_argument(node) + local arg = require_argument(src) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, node.path, 'not a natural number: %s', arg) node.value = as_num end local function init_boolean(node, src, children) - local arg = require_argument(node) + local arg = require_argument(src) if arg == 'true' then node.value = true elseif arg == 'false' then node.value = false else error_with_path(node.path, 'not a valid boolean: %s', arg) end @@ -186,7 +192,7 @@ end -- parse out relevant sub-components and store them as named -- properties on the schema node. local function init_anyxml(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') @@ -197,11 +203,11 @@ local function init_anyxml(node, src, children) node.reference = maybe_child_property(node, 'reference', 'value') end local function init_argument(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.yin_element = maybe_child_property(node, 'yin-element', 'value') end local function init_augment(node, src, children) - node.node_id = require_argument(node) + node.node_id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') @@ -210,11 +216,11 @@ local function init_augment(node, src, children) node.body = collect_data_or_case_children_at_least_1(node) end local function init_belongs_to(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.prefix = require_child(node, 'prefix').value end local function init_case(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') @@ -223,7 +229,7 @@ local function init_case(node, src, children) node.body = collect_body_children(node) end local function init_choice(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.default = maybe_child_property(node, 'default', 'value') @@ -239,7 +245,7 @@ local function init_choice(node, src, children) {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) end local function init_container(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') @@ -253,21 +259,21 @@ local function init_container(node, src, children) node.body = collect_body_children(node) end local function init_extension(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.argument = maybe_child_property(node, 'argument', 'id') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end local function init_feature(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end local function init_grouping(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') @@ -276,19 +282,19 @@ local function init_grouping(node, src, children) node.body = collect_body_children(node) end local function init_identity(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.base = maybe_child_property(node, 'base', 'id') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end local function init_import(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.prefix = require_child_property(node, 'prefix', 'value') node.revision_date = maybe_child_property(node, 'revision-date', 'value') end local function init_include(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.revision_date = maybe_child_property(node, 'revision-date', 'value') end local function init_input(node, src, children) @@ -297,7 +303,7 @@ local function init_input(node, src, children) node.body = collect_body_children_at_least_1(node) end local function init_leaf(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.type = require_child(node, 'type') @@ -311,7 +317,7 @@ local function init_leaf(node, src, children) node.reference = maybe_child_property(node, 'reference', 'value') end local function init_leaf_list(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.type = require_child(node, 'type') @@ -327,12 +333,12 @@ local function init_leaf_list(node, src, children) end local function init_length(node, src, children) -- TODO: parse length arg str - node.value = require_argument(node) + node.value = require_argument(src) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end local function init_list(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') @@ -350,7 +356,7 @@ local function init_list(node, src, children) node.reference = maybe_child_property(node, 'reference', 'value') end local function init_module(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.namespace = require_child_property(node, 'namespace', 'value') node.prefix = require_child_property(node, 'prefix', 'value') @@ -374,10 +380,10 @@ local function init_module(node, src, children) end local function init_namespace(node, src, children) -- TODO: parse uri? - node.value = require_argument(node) + node.value = require_argument(src) end local function init_notification(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') @@ -393,20 +399,20 @@ local function init_output(node, src, children) end local function init_path(node, src, children) -- TODO: parse path string - node.value = require_argument(node) + node.value = require_argument(src) end local function init_pattern(node, src, children) - node.value = require_argument(node) + node.value = require_argument(src) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end local function init_range(node, src, children) - node.value = parse_range(node, require_argument(node)) + node.value = parse_range(node, require_argument(src)) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end local function init_refine(node, src, children) - node.node_id = require_argument(node) + node.node_id = require_argument(src) -- All subnode kinds. node.must = collect_child_properties(children, 'must', 'value') node.config = maybe_child_property(node, 'config', 'value') @@ -423,12 +429,12 @@ local function init_refine(node, src, children) end local function init_revision(node, src, children) -- TODO: parse date - node.value = require_argument(node) + node.value = require_argument(src) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end local function init_rpc(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') @@ -439,7 +445,7 @@ local function init_rpc(node, src, children) node.output = maybe_child(node, 'output') end local function init_type(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.range = maybe_child(node, 'range') node.fraction_digits = maybe_child_property(node, 'fraction-digits', 'value') node.length = maybe_child_property(node, 'length', 'value') @@ -453,7 +459,7 @@ local function init_type(node, src, children) node.bits = collect_children(children, 'bit') end local function init_submodule(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.belongs_to = require_child(node, 'belongs-to') node.imports = collect_children_by_id(node, 'import') @@ -475,7 +481,7 @@ local function init_submodule(node, src, children) node.body = collect_body_children(node) end local function init_typedef(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.type = require_child(node, 'type') node.units = maybe_child_property(node, 'units', 'value') node.default = maybe_child_property(node, 'default', 'value') @@ -484,7 +490,7 @@ local function init_typedef(node, src, children) node.reference = maybe_child_property(node, 'reference', 'value') end local function init_uses(node, src, children) - node.id = require_argument(node) + node.id = require_argument(src) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') @@ -495,7 +501,7 @@ local function init_uses(node, src, children) node.augments = collect_children(children, 'augment') end local function init_value(node, src, children) - local arg = require_argument(node) + local arg = require_argument(src) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num, node.path, 'not an integer: %s', arg) From ef677e0c16cf2269df42232d8cf659957b95d070 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 17:20:41 +0100 Subject: [PATCH 233/631] Further schema parsing refactors. --- src/lib/yang/schema.lua | 248 ++++++++++++++++++++-------------------- 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index e68f23785c..4dd111a135 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -40,7 +40,7 @@ local function parse_node(src, parent_path, order) ret.children = children ret = setmetatable(ret, {__index=Node}) local initialize = initializers[ret.kind] - if initialize then initialize(ret, src, children) end + if initialize then initialize(ret, src.loc, src.argument, children) end -- Strip annotations. ret.path, ret.order, ret.argument_string, ret.children = nil return ret @@ -56,14 +56,14 @@ function parse_children(src, parent_path) return ret end -local function require_argument(src) - return assert_with_loc(src.argument, src.loc, 'missing argument') +local function require_argument(argument, loc) + return assert_with_loc(argument, loc, 'missing argument') end -local function parse_range(node, range) +local function parse_range(loc, range) local function parse_part(part) local l, r = part:match("^%s*([^%.]*)%s*%.%.%s*([^%s]*)%s*$") - assert_with_path(l, node.path, 'bad range component: %s', part) + assert_with_loc(l, loc, 'bad range component: %s', part) if l ~= 'min' then l = util.tointeger(l) end if r ~= 'max' then r = util.tointeger(r) end return { l, r } @@ -71,7 +71,7 @@ local function parse_range(node, range) local parts = range:split("|") local res = {'or'} for part in range:split("|") do table.insert(res, parse_part(part)) end - if #res == 1 then error_with_path(node.path, "empty range", range) + if #res == 1 then error_with_loc(loc, "empty range", range) elseif #res == 2 then return res[2] else return res end end @@ -91,25 +91,25 @@ local function collect_children(children, kinds) return ret end -local function collect_children_by_prop(node, kinds, prop) +local function collect_children_by_prop(loc, children, kinds, prop) local ret = {} - for _, child in ipairs(collect_children(node.children, kinds)) do - assert_with_path(child[prop], node.path, - 'child of kind %s missing prop %s', child.kind, prop) - assert_with_path(not ret[child[prop]], node.path, - 'duplicate %s: %s', prop, child[prop]) + for _, child in ipairs(collect_children(children, kinds)) do + assert_with_loc(child[prop], loc, 'child of kind %s missing prop %s', + child.kind, prop) + assert_with_loc(not ret[child[prop]], loc, 'duplicate %s: %s', + prop, child[prop]) ret[child[prop]] = child end return ret end -local function collect_children_by_id(node, kinds) - return collect_children_by_prop(node, kinds, 'id') +local function collect_children_by_id(loc, children, kinds) + return collect_children_by_prop(loc, children, kinds, 'id') end local function collect_body_children(node) return collect_children_by_id( - node, + node.loc, node.children, {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml'}) end @@ -128,7 +128,7 @@ end local function collect_data_or_case_children_at_least_1(node) local ret = collect_children_by_id( - node, + node.loc, node.children, {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml', 'case'}) if not at_least_one(ret) then @@ -171,18 +171,18 @@ end -- Simple statement kinds with string, natural, or boolean values all -- just initialize by parsing their argument and storing it as the -- "value" property in the schema node. -local function init_string(node, src, children) - node.value = require_argument(src) +local function init_string(node, loc, argument, children) + node.value = require_argument(argument, loc) end -local function init_natural(node, src, children) - local arg = require_argument(src) +local function init_natural(node, loc, argument, children) + local arg = require_argument(argument, loc) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, node.path, 'not a natural number: %s', arg) node.value = as_num end -local function init_boolean(node, src, children) - local arg = require_argument(src) +local function init_boolean(node, loc, argument, children) + local arg = require_argument(argument, loc) if arg == 'true' then node.value = true elseif arg == 'false' then node.value = false else error_with_path(node.path, 'not a valid boolean: %s', arg) end @@ -191,8 +191,8 @@ end -- For all other statement kinds, we have custom initializers that -- parse out relevant sub-components and store them as named -- properties on the schema node. -local function init_anyxml(node, src, children) - node.id = require_argument(src) +local function init_anyxml(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') @@ -202,12 +202,12 @@ local function init_anyxml(node, src, children) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_argument(node, src, children) - node.id = require_argument(src) +local function init_argument(node, loc, argument, children) + node.id = require_argument(argument, loc) node.yin_element = maybe_child_property(node, 'yin-element', 'value') end -local function init_augment(node, src, children) - node.node_id = require_argument(src) +local function init_augment(node, loc, argument, children) + node.node_id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') @@ -215,12 +215,12 @@ local function init_augment(node, src, children) node.reference = maybe_child_property(node, 'reference', 'value') node.body = collect_data_or_case_children_at_least_1(node) end -local function init_belongs_to(node, src, children) - node.id = require_argument(src) +local function init_belongs_to(node, loc, argument, children) + node.id = require_argument(argument, loc) node.prefix = require_child(node, 'prefix').value end -local function init_case(node, src, children) - node.id = require_argument(src) +local function init_case(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') @@ -228,8 +228,8 @@ local function init_case(node, src, children) node.reference = maybe_child_property(node, 'reference', 'value') node.body = collect_body_children(node) end -local function init_choice(node, src, children) - node.id = require_argument(src) +local function init_choice(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.default = maybe_child_property(node, 'default', 'value') @@ -238,14 +238,14 @@ local function init_choice(node, src, children) node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_children_by_id( - node, + loc, children, {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) end -local function init_container(node, src, children) - node.id = require_argument(src) +local function init_container(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') @@ -254,56 +254,56 @@ local function init_container(node, src, children) node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_body_children(node) end -local function init_extension(node, src, children) - node.id = require_argument(src) +local function init_extension(node, loc, argument, children) + node.id = require_argument(argument, loc) node.argument = maybe_child_property(node, 'argument', 'id') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_feature(node, src, children) - node.id = require_argument(src) +local function init_feature(node, loc, argument, children) + node.id = require_argument(argument, loc) node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_grouping(node, src, children) - node.id = require_argument(src) +local function init_grouping(node, loc, argument, children) + node.id = require_argument(argument, loc) node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_body_children(node) end -local function init_identity(node, src, children) - node.id = require_argument(src) +local function init_identity(node, loc, argument, children) + node.id = require_argument(argument, loc) node.base = maybe_child_property(node, 'base', 'id') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_import(node, src, children) - node.id = require_argument(src) +local function init_import(node, loc, argument, children) + node.id = require_argument(argument, loc) node.prefix = require_child_property(node, 'prefix', 'value') node.revision_date = maybe_child_property(node, 'revision-date', 'value') end -local function init_include(node, src, children) - node.id = require_argument(src) +local function init_include(node, loc, argument, children) + node.id = require_argument(argument, loc) node.revision_date = maybe_child_property(node, 'revision-date', 'value') end -local function init_input(node, src, children) - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') +local function init_input(node, loc, argument, children) + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_body_children_at_least_1(node) end -local function init_leaf(node, src, children) - node.id = require_argument(src) +local function init_leaf(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.type = require_child(node, 'type') @@ -316,8 +316,8 @@ local function init_leaf(node, src, children) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_leaf_list(node, src, children) - node.id = require_argument(src) +local function init_leaf_list(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.type = require_child(node, 'type') @@ -331,14 +331,14 @@ local function init_leaf_list(node, src, children) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_length(node, src, children) +local function init_length(node, loc, argument, children) -- TODO: parse length arg str - node.value = require_argument(src) + node.value = require_argument(argument, loc) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_list(node, src, children) - node.id = require_argument(src) +local function init_list(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') @@ -349,70 +349,70 @@ local function init_list(node, src, children) node.max_elements = maybe_child_property(node, 'max-elements', 'value') node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') node.status = maybe_child_property(node, 'status', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_body_children_at_least_1(node) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_module(node, src, children) - node.id = require_argument(src) +local function init_module(node, loc, argument, children) + node.id = require_argument(argument, loc) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.namespace = require_child_property(node, 'namespace', 'value') node.prefix = require_child_property(node, 'prefix', 'value') - node.imports = collect_children_by_prop(node, 'import', 'prefix') - node.includes = collect_children_by_id(node, 'include') + node.imports = collect_children_by_prop(loc, children, 'import', 'prefix') + node.includes = collect_children_by_id(loc, children, 'include') node.organization = maybe_child_property(node, 'organization', 'value') node.contact = maybe_child_property(node, 'contact', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') node.revisions = collect_children(children, 'revision') node.augments = collect_children(children, 'augment') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.features = collect_children_by_id(node, 'feature') - node.extensions = collect_children_by_id(node, 'extension') - node.identities = collect_children_by_id(node, 'identity') - node.rpcs = collect_children_by_id(node, 'rpc') - node.notifications = collect_children_by_id(node, 'notification') - node.deviations = collect_children_by_id(node, 'deviation') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') + node.features = collect_children_by_id(loc, children, 'feature') + node.extensions = collect_children_by_id(loc, children, 'extension') + node.identities = collect_children_by_id(loc, children, 'identity') + node.rpcs = collect_children_by_id(loc, children, 'rpc') + node.notifications = collect_children_by_id(loc, children, 'notification') + node.deviations = collect_children_by_id(loc, children, 'deviation') node.body = collect_body_children(node) end -local function init_namespace(node, src, children) +local function init_namespace(node, loc, argument, children) -- TODO: parse uri? - node.value = require_argument(src) + node.value = require_argument(argument, loc) end -local function init_notification(node, src, children) - node.id = require_argument(src) +local function init_notification(node, loc, argument, children) + node.id = require_argument(argument, loc) node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_body_children(node) end -local function init_output(node, src, children) - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') +local function init_output(node, loc, argument, children) + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_body_children_at_least_1(node) end -local function init_path(node, src, children) +local function init_path(node, loc, argument, children) -- TODO: parse path string - node.value = require_argument(src) + node.value = require_argument(argument, loc) end -local function init_pattern(node, src, children) - node.value = require_argument(src) +local function init_pattern(node, loc, argument, children) + node.value = require_argument(argument, loc) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_range(node, src, children) - node.value = parse_range(node, require_argument(src)) +local function init_range(node, loc, argument, children) + node.value = parse_range(node.loc, require_argument(argument, loc)) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_refine(node, src, children) - node.node_id = require_argument(src) +local function init_refine(node, loc, argument, children) + node.node_id = require_argument(argument, loc) -- All subnode kinds. node.must = collect_child_properties(children, 'must', 'value') node.config = maybe_child_property(node, 'config', 'value') @@ -427,25 +427,25 @@ local function init_refine(node, src, children) node.min_elements = maybe_child_property(node, 'min-elements', 'value') node.max_elements = maybe_child_property(node, 'max-elements', 'value') end -local function init_revision(node, src, children) +local function init_revision(node, loc, argument, children) -- TODO: parse date - node.value = require_argument(src) + node.value = require_argument(argument, loc) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_rpc(node, src, children) - node.id = require_argument(src) +local function init_rpc(node, loc, argument, children) + node.id = require_argument(argument, loc) node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') node.input = maybe_child(node, 'input') node.output = maybe_child(node, 'output') end -local function init_type(node, src, children) - node.id = require_argument(src) +local function init_type(node, loc, argument, children) + node.id = require_argument(argument, loc) node.range = maybe_child(node, 'range') node.fraction_digits = maybe_child_property(node, 'fraction-digits', 'value') node.length = maybe_child_property(node, 'length', 'value') @@ -458,30 +458,30 @@ local function init_type(node, src, children) node.union = collect_children(children, 'type') node.bits = collect_children(children, 'bit') end -local function init_submodule(node, src, children) - node.id = require_argument(src) +local function init_submodule(node, loc, argument, children) + node.id = require_argument(argument, loc) node.yang_version = maybe_child_property(node, 'yang-version', 'value') node.belongs_to = require_child(node, 'belongs-to') - node.imports = collect_children_by_id(node, 'import') - node.includes = collect_children_by_id(node, 'include') + node.imports = collect_children_by_id(loc, children, 'import') + node.includes = collect_children_by_id(loc, children, 'include') node.organization = maybe_child_property(node, 'organization', 'value') node.contact = maybe_child_property(node, 'contact', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') node.revisions = collect_children(children, 'revision') node.augments = collect_children(children, 'augment') - node.typedefs = collect_children_by_id(node, 'typedef') - node.groupings = collect_children_by_id(node, 'grouping') - node.features = collect_children_by_id(node, 'feature') - node.extensions = collect_children_by_id(node, 'extension') - node.identities = collect_children_by_id(node, 'identity') - node.rpcs = collect_children_by_id(node, 'rpc') - node.notifications = collect_children_by_id(node, 'notification') - node.deviations = collect_children_by_id(node, 'deviation') + node.typedefs = collect_children_by_id(loc, children, 'typedef') + node.groupings = collect_children_by_id(loc, children, 'grouping') + node.features = collect_children_by_id(loc, children, 'feature') + node.extensions = collect_children_by_id(loc, children, 'extension') + node.identities = collect_children_by_id(loc, children, 'identity') + node.rpcs = collect_children_by_id(loc, children, 'rpc') + node.notifications = collect_children_by_id(loc, children, 'notification') + node.deviations = collect_children_by_id(loc, children, 'deviation') node.body = collect_body_children(node) end -local function init_typedef(node, src, children) - node.id = require_argument(src) +local function init_typedef(node, loc, argument, children) + node.id = require_argument(argument, loc) node.type = require_child(node, 'type') node.units = maybe_child_property(node, 'units', 'value') node.default = maybe_child_property(node, 'default', 'value') @@ -489,19 +489,19 @@ local function init_typedef(node, src, children) node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') end -local function init_uses(node, src, children) - node.id = require_argument(src) +local function init_uses(node, loc, argument, children) + node.id = require_argument(argument, loc) node.when = maybe_child_property(node, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.status = maybe_child_property(node, 'status', 'value') node.description = maybe_child_property(node, 'description', 'value') node.reference = maybe_child_property(node, 'reference', 'value') - node.typedefs = collect_children_by_id(node, 'typedef') + node.typedefs = collect_children_by_id(loc, children, 'typedef') node.refines = collect_children(children, 'refine') node.augments = collect_children(children, 'augment') end -local function init_value(node, src, children) - local arg = require_argument(src) +local function init_value(node, loc, argument, children) + local arg = require_argument(argument, loc) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num, node.path, 'not an integer: %s', arg) From de24664dbb1533055141a83675817a2fd8afefc6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 17:40:58 +0100 Subject: [PATCH 234/631] Schema parsing refactors redux. --- src/lib/yang/schema.lua | 384 ++++++++++++++++++++-------------------- 1 file changed, 191 insertions(+), 193 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 4dd111a135..0d2c7fcb71 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -34,15 +34,13 @@ local function parse_node(src, parent_path, order) else ret.path = ret.kind end - ret.order = order - ret.argument_string = src.argument local children = parse_children(src, ret.path) ret.children = children ret = setmetatable(ret, {__index=Node}) local initialize = initializers[ret.kind] if initialize then initialize(ret, src.loc, src.argument, children) end -- Strip annotations. - ret.path, ret.order, ret.argument_string, ret.children = nil + ret.path, ret.children = nil return ret end @@ -56,7 +54,7 @@ function parse_children(src, parent_path) return ret end -local function require_argument(argument, loc) +local function require_argument(loc, argument) return assert_with_loc(argument, loc, 'missing argument') end @@ -107,9 +105,9 @@ local function collect_children_by_id(loc, children, kinds) return collect_children_by_prop(loc, children, kinds, 'id') end -local function collect_body_children(node) +local function collect_body_children(loc, children) return collect_children_by_id( - node.loc, node.children, + loc, children, {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml'}) end @@ -118,21 +116,21 @@ local function at_least_one(tab) return false end -local function collect_body_children_at_least_1(node) - local ret = collect_body_children(node) +local function collect_body_children_at_least_1(loc, children) + local ret = collect_body_children(loc, children) if not at_least_one(ret) then - error_with_path(node.path, "missing data statements") + error_with_loc(loc, "missing data statements") end return ret end -local function collect_data_or_case_children_at_least_1(node) +local function collect_data_or_case_children_at_least_1(loc, children) local ret = collect_children_by_id( - node.loc, node.children, + loc, children, {'container', 'leaf', 'list', 'leaf-list', 'uses', 'choice', 'anyxml', 'case'}) if not at_least_one(ret) then - error_with_path(node.path, "missing data statements") + error_with_loc(loc, "missing data statements") end return ret end @@ -145,99 +143,99 @@ local function collect_child_properties(children, kind, field) return ret end -local function maybe_child(node, kind) - local children = collect_children(node.children, kind) +local function maybe_child(loc, children, kind) + local children = collect_children(children, kind) if #children > 1 then - error_with_path(node.path, 'expected at most one child of type %s', kind) + error_with_loc(loc, 'expected at most one child of type %s', kind) end return children[1] end -local function maybe_child_property(node, kind, prop) - local child = maybe_child(node, kind) +local function maybe_child_property(loc, children, kind, prop) + local child = maybe_child(loc, children, kind) if child then return child[prop] end end -local function require_child(node, kind) - local child = maybe_child(node, kind) +local function require_child(loc, children, kind) + local child = maybe_child(loc, children, kind) if child then return child end - error_with_path(node.path, 'missing child of type %s', kind) + error_with_loc(loc, 'missing child of type %s', kind) end -local function require_child_property(node, kind, prop) - return require_child(node, kind)[prop] +local function require_child_property(loc, children, kind, prop) + return require_child(loc, children, kind)[prop] end -- Simple statement kinds with string, natural, or boolean values all -- just initialize by parsing their argument and storing it as the -- "value" property in the schema node. local function init_string(node, loc, argument, children) - node.value = require_argument(argument, loc) + node.value = require_argument(loc, argument) end local function init_natural(node, loc, argument, children) - local arg = require_argument(argument, loc) + local arg = require_argument(loc, argument) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, node.path, 'not a natural number: %s', arg) node.value = as_num end local function init_boolean(node, loc, argument, children) - local arg = require_argument(argument, loc) + local arg = require_argument(loc, argument) if arg == 'true' then node.value = true elseif arg == 'false' then node.value = false - else error_with_path(node.path, 'not a valid boolean: %s', arg) end + else error_with_loc(loc, 'not a valid boolean: %s', arg) end end -- For all other statement kinds, we have custom initializers that -- parse out relevant sub-components and store them as named -- properties on the schema node. local function init_anyxml(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.config = maybe_child_property(loc, children, 'config', 'value') + node.mandatory = maybe_child_property(loc, children, 'mandatory', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_argument(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.yin_element = maybe_child_property(node, 'yin-element', 'value') + node.id = require_argument(loc, argument) + node.yin_element = maybe_child_property(loc, children, 'yin-element', 'value') end local function init_augment(node, loc, argument, children) - node.node_id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.node_id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.body = collect_data_or_case_children_at_least_1(node) + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') + node.body = collect_data_or_case_children_at_least_1(loc, children) end local function init_belongs_to(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.prefix = require_child(node, 'prefix').value + node.id = require_argument(loc, argument) + node.prefix = require_child(loc, children, 'prefix').value end local function init_case(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') - node.body = collect_body_children(node) + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') + node.body = collect_body_children(loc, children) end local function init_choice(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.default = maybe_child_property(node, 'default', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.default = maybe_child_property(loc, children, 'default', 'value') + node.config = maybe_child_property(loc, children, 'config', 'value') + node.mandatory = maybe_child_property(loc, children, 'mandatory', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') node.body = collect_children_by_id( @@ -245,127 +243,127 @@ local function init_choice(node, loc, argument, children) {'container', 'leaf', 'leaf-list', 'list', 'anyxml', 'case'}) end local function init_container(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') - node.presence = maybe_child_property(node, 'presence', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.presence = maybe_child_property(loc, children, 'presence', 'value') + node.config = maybe_child_property(loc, children, 'config', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') - node.body = collect_body_children(node) + node.body = collect_body_children(loc, children) end local function init_extension(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.argument = maybe_child_property(node, 'argument', 'id') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.id = require_argument(loc, argument) + node.argument = maybe_child_property(loc, children, 'argument', 'id') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_feature(node, loc, argument, children) - node.id = require_argument(argument, loc) + node.id = require_argument(loc, argument) node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_grouping(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.id = require_argument(loc, argument) + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') - node.body = collect_body_children(node) + node.body = collect_body_children(loc, children) end local function init_identity(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.base = maybe_child_property(node, 'base', 'id') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.id = require_argument(loc, argument) + node.base = maybe_child_property(loc, children, 'base', 'id') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_import(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.prefix = require_child_property(node, 'prefix', 'value') - node.revision_date = maybe_child_property(node, 'revision-date', 'value') + node.id = require_argument(loc, argument) + node.prefix = require_child_property(loc, children, 'prefix', 'value') + node.revision_date = maybe_child_property(loc, children, 'revision-date', 'value') end local function init_include(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.revision_date = maybe_child_property(node, 'revision-date', 'value') + node.id = require_argument(loc, argument) + node.revision_date = maybe_child_property(loc, children, 'revision-date', 'value') end local function init_input(node, loc, argument, children) node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') - node.body = collect_body_children_at_least_1(node) + node.body = collect_body_children_at_least_1(loc, children) end local function init_leaf(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.type = require_child(node, 'type') - node.units = maybe_child_property(node, 'units', 'value') + node.type = require_child(loc, children, 'type') + node.units = maybe_child_property(loc, children, 'units', 'value') node.must = collect_child_properties(children, 'must', 'value') - node.default = maybe_child_property(node, 'default', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.default = maybe_child_property(loc, children, 'default', 'value') + node.config = maybe_child_property(loc, children, 'config', 'value') + node.mandatory = maybe_child_property(loc, children, 'mandatory', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_leaf_list(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.type = require_child(node, 'type') - node.units = maybe_child_property(node, 'units', 'value') + node.type = require_child(loc, children, 'type') + node.units = maybe_child_property(loc, children, 'units', 'value') node.must = collect_child_properties(children, 'must', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.min_elements = maybe_child_property(node, 'min-elements', 'value') - node.max_elements = maybe_child_property(node, 'max-elements', 'value') - node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.config = maybe_child_property(loc, children, 'config', 'value') + node.min_elements = maybe_child_property(loc, children, 'min-elements', 'value') + node.max_elements = maybe_child_property(loc, children, 'max-elements', 'value') + node.ordered_by = maybe_child_property(loc, children, 'ordered-by', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_length(node, loc, argument, children) -- TODO: parse length arg str - node.value = require_argument(argument, loc) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.value = require_argument(loc, argument) + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_list(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') node.must = collect_child_properties(children, 'must', 'value') - node.key = maybe_child_property(node, 'key', 'value') + node.key = maybe_child_property(loc, children, 'key', 'value') node.unique = collect_child_properties(children, 'unique', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.min_elements = maybe_child_property(node, 'min-elements', 'value') - node.max_elements = maybe_child_property(node, 'max-elements', 'value') - node.ordered_by = maybe_child_property(node, 'ordered-by', 'value') - node.status = maybe_child_property(node, 'status', 'value') + node.config = maybe_child_property(loc, children, 'config', 'value') + node.min_elements = maybe_child_property(loc, children, 'min-elements', 'value') + node.max_elements = maybe_child_property(loc, children, 'max-elements', 'value') + node.ordered_by = maybe_child_property(loc, children, 'ordered-by', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') - node.body = collect_body_children_at_least_1(node) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.body = collect_body_children_at_least_1(loc, children) + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_module(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.yang_version = maybe_child_property(node, 'yang-version', 'value') - node.namespace = require_child_property(node, 'namespace', 'value') - node.prefix = require_child_property(node, 'prefix', 'value') + node.id = require_argument(loc, argument) + node.yang_version = maybe_child_property(loc, children, 'yang-version', 'value') + node.namespace = require_child_property(loc, children, 'namespace', 'value') + node.prefix = require_child_property(loc, children, 'prefix', 'value') node.imports = collect_children_by_prop(loc, children, 'import', 'prefix') node.includes = collect_children_by_id(loc, children, 'include') - node.organization = maybe_child_property(node, 'organization', 'value') - node.contact = maybe_child_property(node, 'contact', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.organization = maybe_child_property(loc, children, 'organization', 'value') + node.contact = maybe_child_property(loc, children, 'contact', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.revisions = collect_children(children, 'revision') node.augments = collect_children(children, 'augment') node.typedefs = collect_children_by_id(loc, children, 'typedef') @@ -376,98 +374,98 @@ local function init_module(node, loc, argument, children) node.rpcs = collect_children_by_id(loc, children, 'rpc') node.notifications = collect_children_by_id(loc, children, 'notification') node.deviations = collect_children_by_id(loc, children, 'deviation') - node.body = collect_body_children(node) + node.body = collect_body_children(loc, children) end local function init_namespace(node, loc, argument, children) -- TODO: parse uri? - node.value = require_argument(argument, loc) + node.value = require_argument(loc, argument) end local function init_notification(node, loc, argument, children) - node.id = require_argument(argument, loc) + node.id = require_argument(loc, argument) node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') - node.body = collect_body_children(node) + node.body = collect_body_children(loc, children) end local function init_output(node, loc, argument, children) node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') - node.body = collect_body_children_at_least_1(node) + node.body = collect_body_children_at_least_1(loc, children) end local function init_path(node, loc, argument, children) -- TODO: parse path string - node.value = require_argument(argument, loc) + node.value = require_argument(loc, argument) end local function init_pattern(node, loc, argument, children) - node.value = require_argument(argument, loc) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.value = require_argument(loc, argument) + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_range(node, loc, argument, children) - node.value = parse_range(node.loc, require_argument(argument, loc)) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.value = parse_range(loc, require_argument(loc, argument)) + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_refine(node, loc, argument, children) - node.node_id = require_argument(argument, loc) + node.node_id = require_argument(loc, argument) -- All subnode kinds. node.must = collect_child_properties(children, 'must', 'value') - node.config = maybe_child_property(node, 'config', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.config = maybe_child_property(loc, children, 'config', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') -- Containers. - node.presence = maybe_child_property(node, 'presence', 'value') + node.presence = maybe_child_property(loc, children, 'presence', 'value') -- Leaves, choice, and (for mandatory) anyxml. - node.default = maybe_child_property(node, 'default', 'value') - node.mandatory = maybe_child_property(node, 'mandatory', 'value') + node.default = maybe_child_property(loc, children, 'default', 'value') + node.mandatory = maybe_child_property(loc, children, 'mandatory', 'value') -- Leaf lists and lists. - node.min_elements = maybe_child_property(node, 'min-elements', 'value') - node.max_elements = maybe_child_property(node, 'max-elements', 'value') + node.min_elements = maybe_child_property(loc, children, 'min-elements', 'value') + node.max_elements = maybe_child_property(loc, children, 'max-elements', 'value') end local function init_revision(node, loc, argument, children) -- TODO: parse date - node.value = require_argument(argument, loc) - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.value = require_argument(loc, argument) + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_rpc(node, loc, argument, children) - node.id = require_argument(argument, loc) + node.id = require_argument(loc, argument) node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.typedefs = collect_children_by_id(loc, children, 'typedef') node.groupings = collect_children_by_id(loc, children, 'grouping') - node.input = maybe_child(node, 'input') - node.output = maybe_child(node, 'output') + node.input = maybe_child(loc, children, 'input') + node.output = maybe_child(loc, children, 'output') end local function init_type(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.range = maybe_child(node, 'range') - node.fraction_digits = maybe_child_property(node, 'fraction-digits', 'value') - node.length = maybe_child_property(node, 'length', 'value') + node.id = require_argument(loc, argument) + node.range = maybe_child(loc, children, 'range') + node.fraction_digits = maybe_child_property(loc, children, 'fraction-digits', 'value') + node.length = maybe_child_property(loc, children, 'length', 'value') node.patterns = collect_children(children, 'pattern') node.enums = collect_children(children, 'enum') -- !!! path - node.leafref = maybe_child_property(node, 'path', 'value') + node.leafref = maybe_child_property(loc, children, 'path', 'value') node.require_instances = collect_children(children, 'require-instance') - node.identityref = maybe_child_property(node, 'base', 'value') + node.identityref = maybe_child_property(loc, children, 'base', 'value') node.union = collect_children(children, 'type') node.bits = collect_children(children, 'bit') end local function init_submodule(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.yang_version = maybe_child_property(node, 'yang-version', 'value') - node.belongs_to = require_child(node, 'belongs-to') + node.id = require_argument(loc, argument) + node.yang_version = maybe_child_property(loc, children, 'yang-version', 'value') + node.belongs_to = require_child(loc, children, 'belongs-to') node.imports = collect_children_by_id(loc, children, 'import') node.includes = collect_children_by_id(loc, children, 'include') - node.organization = maybe_child_property(node, 'organization', 'value') - node.contact = maybe_child_property(node, 'contact', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.organization = maybe_child_property(loc, children, 'organization', 'value') + node.contact = maybe_child_property(loc, children, 'contact', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.revisions = collect_children(children, 'revision') node.augments = collect_children(children, 'augment') node.typedefs = collect_children_by_id(loc, children, 'typedef') @@ -478,30 +476,30 @@ local function init_submodule(node, loc, argument, children) node.rpcs = collect_children_by_id(loc, children, 'rpc') node.notifications = collect_children_by_id(loc, children, 'notification') node.deviations = collect_children_by_id(loc, children, 'deviation') - node.body = collect_body_children(node) + node.body = collect_body_children(loc, children) end local function init_typedef(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.type = require_child(node, 'type') - node.units = maybe_child_property(node, 'units', 'value') - node.default = maybe_child_property(node, 'default', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.id = require_argument(loc, argument) + node.type = require_child(loc, children, 'type') + node.units = maybe_child_property(loc, children, 'units', 'value') + node.default = maybe_child_property(loc, children, 'default', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') end local function init_uses(node, loc, argument, children) - node.id = require_argument(argument, loc) - node.when = maybe_child_property(node, 'when', 'value') + node.id = require_argument(loc, argument) + node.when = maybe_child_property(loc, children, 'when', 'value') node.if_features = collect_child_properties(children, 'if-feature', 'value') - node.status = maybe_child_property(node, 'status', 'value') - node.description = maybe_child_property(node, 'description', 'value') - node.reference = maybe_child_property(node, 'reference', 'value') + node.status = maybe_child_property(loc, children, 'status', 'value') + node.description = maybe_child_property(loc, children, 'description', 'value') + node.reference = maybe_child_property(loc, children, 'reference', 'value') node.typedefs = collect_children_by_id(loc, children, 'typedef') node.refines = collect_children(children, 'refine') node.augments = collect_children(children, 'augment') end local function init_value(node, loc, argument, children) - local arg = require_argument(argument, loc) + local arg = require_argument(loc, argument) local as_num = tonumber(arg) assert_with_path(as_num and math.floor(as_num) == as_num, node.path, 'not an integer: %s', arg) From 423a032a6f2d6b421120958a1fbad3fd02cf371c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 17:45:15 +0100 Subject: [PATCH 235/631] No more annotation of parser source terms. --- src/lib/yang/schema.lua | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 0d2c7fcb71..9cb2d47564 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -11,13 +11,6 @@ local function assert_with_loc(expr, loc, msg, ...) if not expr then error_with_path(loc, msg, ...) end return expr end -local function error_with_path(path, msg, ...) - error(string.format("%s: "..msg, path, ...)) -end -local function assert_with_path(expr, path, msg, ...) - if not expr then error_with_path(path, msg, ...) end - return expr -end -- (kind -> (function(Node) -> value)) local initializers = {} @@ -25,29 +18,19 @@ local function declare_initializer(init, ...) for _, keyword in ipairs({...}) do initializers[keyword] = init end end -local Node = {} -local function parse_node(src, parent_path, order) +local function parse_node(src) local ret = {} ret.kind = assert(src.keyword, 'missing keyword') - if parent_path then - ret.path = parent_path..'.'..ret.kind - else - ret.path = ret.kind - end - local children = parse_children(src, ret.path) - ret.children = children - ret = setmetatable(ret, {__index=Node}) + local children = parse_children(src) local initialize = initializers[ret.kind] if initialize then initialize(ret, src.loc, src.argument, children) end - -- Strip annotations. - ret.path, ret.children = nil return ret end -function parse_children(src, parent_path) +function parse_children(src) local ret = {} for i, statement in ipairs(src.statements or {}) do - local child = parse_node(statement, parent_path, i) + local child = parse_node(statement) if not ret[child.kind] then ret[child.kind] = {} end table.insert(ret[child.kind], child) end @@ -175,8 +158,8 @@ end local function init_natural(node, loc, argument, children) local arg = require_argument(loc, argument) local as_num = tonumber(arg) - assert_with_path(as_num and math.floor(as_num) == as_num and as_num >= 0, - node.path, 'not a natural number: %s', arg) + assert_with_loc(as_num and math.floor(as_num) == as_num and as_num >= 0, + loc, 'not a natural number: %s', arg) node.value = as_num end local function init_boolean(node, loc, argument, children) @@ -501,8 +484,8 @@ end local function init_value(node, loc, argument, children) local arg = require_argument(loc, argument) local as_num = tonumber(arg) - assert_with_path(as_num and math.floor(as_num) == as_num, - node.path, 'not an integer: %s', arg) + assert_with_loc(as_num and math.floor(as_num) == as_num, + loc, 'not an integer: %s', arg) node.value = as_num end From 5916b09081ad4b721ac3fee634c3003223e47e13 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 17:46:03 +0100 Subject: [PATCH 236/631] Fix assert_with_loc bug. --- src/lib/yang/schema.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 9cb2d47564..88ebc51e40 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -8,7 +8,7 @@ local function error_with_loc(loc, msg, ...) error(string.format("%s: "..msg, loc, ...)) end local function assert_with_loc(expr, loc, msg, ...) - if not expr then error_with_path(loc, msg, ...) end + if not expr then error_with_loc(loc, msg, ...) end return expr end From 1811923a721f639a0f3b4cfd082d695decaba017 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 15 Nov 2016 18:18:32 +0100 Subject: [PATCH 237/631] Implement load-config command Had to remove lwaftr channels because the lwaftr carps if the channel already exists, when the lwaftr is being reloaded; but that's just a sign that everything works and that old code is no longer needed. Wooooooo hoooooo! --- src/apps/config/leader.lua | 8 + src/apps/lwaftr/channel.lua | 211 ------------------ src/apps/lwaftr/lwaftr.lua | 26 --- src/apps/lwaftr/messages.lua | 7 - src/lib/yang/data.lua | 7 +- src/lib/yang/snabb-config-leader-v1.yang | 8 + src/program/config/load_config/README.inc | 15 ++ .../config/load_config/load_config.lua | 89 ++++++++ 8 files changed, 126 insertions(+), 245 deletions(-) delete mode 100644 src/apps/lwaftr/channel.lua delete mode 100644 src/apps/lwaftr/messages.lua create mode 100644 src/program/config/load_config/README.inc create mode 100644 src/program/config/load_config/load_config.lua diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 3ff6d48b7a..3776594c95 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -91,6 +91,7 @@ end function Leader:rpc_get_config (args) -- FIXME: Push more of this to a lib. + assert(args.schema == self.schema_name) assert(args.path == '/') local schema = yang.load_schema_by_name(self.schema_name) local grammar = data.data_grammar_from_schema(schema) @@ -99,6 +100,13 @@ function Leader:rpc_get_config (args) return { config = config_str } end +function Leader:rpc_load_config (args) + assert(args.schema == self.schema_name) + local config = yang.load_data_for_schema_by_name(args.schema, args.config) + self:reset_configuration(config) + return {} +end + function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end diff --git a/src/apps/lwaftr/channel.lua b/src/apps/lwaftr/channel.lua deleted file mode 100644 index 010ead18d1..0000000000 --- a/src/apps/lwaftr/channel.lua +++ /dev/null @@ -1,211 +0,0 @@ --- Channels --- --- A channel is a way for different threads or processes to communicate. --- Channels are backed by a ring buffer that is mapped into shared --- memory. Access to a channel will never block or cause a system call. --- Readers and writers have to agree ahead of time on how to interpret --- the messages that are written to a channel. - -module(..., package.seeall) - -local ffi = require('ffi') -local S = require("syscall") -local lib = require('core.lib') - -root = "/var/run/snabb" - -local ring_buffer_t = ffi.typeof([[struct { - uint32_t read; - uint32_t write; - uint32_t size; - uint32_t flags; - uint8_t buf[?]; -}]]) - --- Make directories needed for a named object. --- Given the name "foo/bar/baz" create /var/run/foo and /var/run/foo/bar. -local function mkdir_p (name) - -- Create root with mode "rwxrwxrwt" (R/W for all and sticky) if it - -- does not exist yet. - if not S.stat(root) then - local mask = S.umask(0) - local status, err = S.mkdir(root, "01777") - assert(status, ("Unable to create %s: %s"):format( - root, tostring(err or "unspecified error"))) - S.umask(mask) - end - -- Create sub directories - local dir = root - name:gsub("([^/]+)", - function (x) S.mkdir(dir, "rwxu") dir = dir.."/"..x end) -end - -local function create_ring_buffer (name, size) - local tail = tostring(S.getpid())..'/channels/'..name - local path = root..'/'..tail - mkdir_p(tail) - local fd, err = S.open(path, "creat, rdwr, excl", '0664') - if not fd then - err = tostring(err or "unknown error") - error('error creating file "'..path..'": '..err) - end - local len = ffi.sizeof(ring_buffer_t, size) - assert(fd:ftruncate(len), "ring buffer: ftruncate failed") - local mem, err = S.mmap(nil, len, "read, write", "shared", fd, 0) - fd:close() - if mem == nil then error("mmap failed: " .. tostring(err)) end - mem = ffi.cast(ffi.typeof("$*", ring_buffer_t), mem) - ffi.gc(mem, function (ptr) S.munmap(ptr, len) end) - mem.size = size - return mem -end - -local function open_ring_buffer (pid, name) - local path = root..'/'..tostring(pid)..'/channels/'..name - local fd, err = S.open(path, "rdwr") - if not fd then - err = tostring(err or "unknown error") - error('error opening file "'..path..'": '..err) - end - local stat = S.fstat(fd) - local len = stat and stat.size - if len < ffi.sizeof(ring_buffer_t, 0) then - error("unexpected size for ring buffer") - end - local mem, err = S.mmap(nil, len, "read, write", "shared", fd, 0) - fd:close() - if mem == nil then error("mmap failed: " .. tostring(err)) end - mem = ffi.cast(ffi.typeof("$*", ring_buffer_t), mem) - ffi.gc(mem, function (ptr) S.munmap(ptr, len) end) - if len ~= ffi.sizeof(ring_buffer_t, mem.size) then - error("unexpected ring buffer size: "..tostring(len)) - end - return mem -end - -local function to_uint32 (num) - local buf = ffi.new('uint32_t[1]') - buf[0] = num - return buf[0] -end - -local function read_avail (ring) - lib.compiler_barrier() - return to_uint32(ring.write - ring.read) -end - -local function write_avail (ring) - return ring.size - read_avail(ring) -end - --- The coordination needed between the reader and the writer is that: --- --- 1. If the reader sees a a bumped write pointer, that the data written --- to the ring buffer will be available to the reader, i.e. the writer --- has done whatever is needed to synchronize the data. --- --- 2. It should be possible for the reader to update the read pointer --- without stompling other memory, notably the write pointer. --- --- 3. It should be possible for the writer to update the write pointer --- without stompling other memory, notably the read pointer. --- --- 4. Updating a write pointer or a read pointer should eventually be --- visible to the reader or writer, respectively. --- --- The full memory barrier after updates to the read or write pointer --- ensures (1). The x86 memory model, and the memory model of C11, --- guarantee (2) and (3). For (4), the memory barrier on the writer --- side ensures that updates to the read or write pointers are --- eventually visible to other CPUs, but we also have to insert a --- compiler barrier before reading them to prevent LuaJIT from caching --- their value somewhere else, like in a register. See --- https://www.kernel.org/doc/Documentation/memory-barriers.txt for more --- discussion on memory models, and --- http://www.freelists.org/post/luajit/Compiler-loadstore-barrier-volatile-pointer-barriers-in-general,3 --- for more on compiler barriers in LuaJIT. --- --- If there are multiple readers or writers, they should serialize their --- accesses through some other mechanism. --- -local function put_bytes (ring, bytes, count) - local new_write_avail = write_avail(ring) - count - if new_write_avail < 0 then return new_write_avail end - local start = ring.write % ring.size - if start + count > ring.size then - local head = ring.size - start - ffi.copy(ring.buf + start, bytes, head) - ffi.copy(ring.buf, bytes + head, count - head) - else - ffi.copy(ring.buf + start, bytes, count) - end - ring.write = ring.write + count - ffi.C.full_memory_barrier() - return new_write_avail -end - -local function get_bytes (ring, count, buf) - if read_avail(ring) < count then return nil end - buf = buf or ffi.new('uint8_t[?]', count) - local start = ring.read % ring.size - if start + count > ring.size then - ffi.copy(buf, ring.buf + start, ring.size - start) - ffi.copy(buf, ring.buf, count - ring.size) - else - ffi.copy(buf, ring.buf + start, count) - end - ring.read = ring.read + count - ffi.C.full_memory_barrier() - return buf -end - -Channel = {} - -local default_buffer_size = 32 -function create(name, type, size) - local ret = {} - size = size or default_buffer_size - ret.ring_buffer = create_ring_buffer(name, ffi.sizeof(type) * size) - ret.type = type - ret.type_ptr = ffi.typeof('$*', type) - return setmetatable(ret, {__index=Channel}) -end - -function open(pid, name, type) - local ret = {} - ret.ring_buffer = open_ring_buffer(pid, name) - if ret.ring_buffer.size % ffi.sizeof(type) ~= 0 then - error ("Unexpected channel size: "..ret.ring_buffer.size) - end - ret.type = type - return setmetatable(ret, {__index=Channel}) -end - -function Channel:put(...) - local val = self.type(...) - return put_bytes(self.ring_buffer, val, ffi.sizeof(self.type)) >= 0 -end - -function Channel:pop() - local ret = get_bytes(self.ring_buffer, ffi.sizeof(self.type)) - if ret then return ffi.cast(self.type_ptr, ret) end -end - -function selftest() - print('selftest: channel') - local msg_t = ffi.typeof('struct { uint8_t a; uint8_t b; }') - local ch = create('test/control', msg_t, 16) - for i=1,16 do assert(ch:put({i, i+16})) end - assert(not ch:put({0,0})) - local function assert_pop(a, b) - local msg = assert(ch:pop()) - assert(msg.a == a) - assert(msg.b == b) - end - assert_pop(1, 17) - assert(ch:put({17, 33})) - assert(not ch:put({0,0})) - for i=2,17 do assert_pop(i, i+16) end - assert(not ch:pop()) - print('selftest: channel ok') -end diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 118d670456..0bb9f3cc74 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -9,9 +9,6 @@ local lwdebug = require("apps.lwaftr.lwdebug") local lwheader = require("apps.lwaftr.lwheader") local lwutil = require("apps.lwaftr.lwutil") -local channel = require("apps.lwaftr.channel") -local messages = require("apps.lwaftr.messages") - local checksum = require("lib.checksum") local ethernet = require("lib.protocol.ethernet") local counter = require("core.counter") @@ -258,7 +255,6 @@ function LwAftr:new(conf) conf.external_interface.next_hop.mac = ethernet:pton('00:00:00:00:00:00') end - o.control = channel.create('lwaftr/control', messages.lwaftr_message_t) o.counters = lwcounter.init_counters() o.transmit_icmpv6_reply = init_transmit_icmpv6_reply( @@ -807,28 +803,6 @@ function LwAftr:push () local o4, o6 = self.output.v4, self.output.v6 self.o4, self.o6 = o4, o6 - do - local msg = self.control:pop() - if msg then - if msg.kind == messages.lwaftr_message_reload then - print('Reloading binding table.') - print('FIXME: Out to lunch, back shortly') - if false then - self.binding_table = bt.load(self.conf.binding_table) - end - -- We don't know why yet, but something about reloading a - -- binding table makes LuaJIT switch to side traces instead - -- of main traces. Very weird. Flushing the JIT state - -- fixes it, but it's quite a big hammer! - require('jit').flush() - elseif msg.kind == messages.lwaftr_message_dump_config then - dump.dump_configuration(self) - else - print('Unhandled message: '..tostring(msg)) - end - end - end - for _ = 1, link.nreadable(i6) do -- Decapsulate incoming IPv6 packets from the B4 interface and -- push them out the V4 link, unless they need hairpinning, in diff --git a/src/apps/lwaftr/messages.lua b/src/apps/lwaftr/messages.lua deleted file mode 100644 index 666102da49..0000000000 --- a/src/apps/lwaftr/messages.lua +++ /dev/null @@ -1,7 +0,0 @@ -module(..., package.seeall) - -local ffi = require('ffi') - -lwaftr_message_t = ffi.typeof('struct { uint8_t kind; }') -lwaftr_message_reload = 1 -lwaftr_message_dump_config = 2 diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 70f1ddcf93..3a48bb0b6e 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -114,7 +114,12 @@ function rpc_grammar_from_schema(schema) for _,prop in ipairs({'input', 'output'}) do grammar[prop] = { type="sequence", members={} } for k,rpc in pairs(schema.rpcs) do - grammar[prop].members[k] = data_grammar_from_schema(rpc[prop]) + local node = rpc[prop] + if node then + grammar[prop].members[k] = data_grammar_from_schema(node) + else + grammar[prop].members[k] = {type="struct", members={}} + end end end return grammar diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index a032adee1a..1aea50ce65 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -22,4 +22,12 @@ module snabb-config-leader-v1 { leaf config { type string; } } } + + rpc load-config { + input { + leaf schema { type string; mandatory true; } + leaf revision { type string; } + leaf config { type string; mandatory true; } + } + } } diff --git a/src/program/config/load_config/README.inc b/src/program/config/load_config/README.inc new file mode 100644 index 0000000000..f5e7c99d79 --- /dev/null +++ b/src/program/config/load_config/README.inc @@ -0,0 +1,15 @@ +Usage: + snabb config load-config [OPTIONS] ID FILE + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/load_config/load_config.lua b/src/program/config/load_config/load_config.lua new file mode 100644 index 0000000000..5aeedba70f --- /dev/null +++ b/src/program/config/load_config/load_config.lua @@ -0,0 +1,89 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local S = require("syscall") +local ffi = require("ffi") +local lib = require("core.lib") +local shm = require("core.shm") +local rpc = require("lib.yang.rpc") +local yang = require("lib.yang.yang") +local data = require("lib.yang.data") + +-- Number of spaces a tab should consist of when indenting config. +local tab_spaces = 2 + +local function show_usage(status) + print(require("program.config.load_config.README_inc")) + main.exit(status) +end + +local function parse_args(args) + local schema_name, revision_date + local handlers = {} + function handlers.h() show_usage(0) end + function handlers.s(arg) schema_name = arg end + function handlers.r(arg) revision_date = arg end + args = lib.dogetopt(args, handlers, "h:s:r", + {help="h", ['schema-name']="s", schema="s", + ['revision-date']="r", revision="r"}) + if not schema_name then show_usage(1) end + if #args < 2 or #args > 2 then show_usage(1) end + local instance_id, config_file = unpack(args) + return schema_name, revision_date, instance_id, config_file +end + +local function read_length(socket) + local len = 0 + while true do + local ch = assert(socket:read(nil, 1)) + assert(ch ~= '', 'short read') + if ch == '\n' then return len end + assert(tonumber(ch), 'not a number: '..ch) + len = len * 10 + tonumber(ch) + assert(len < 1e8, 'length too long: '..len) + end +end + +local function read_msg(socket, len) + local buf = ffi.new('uint8_t[?]', len) + local pos = 0 + while pos < len do + local count = assert(socket:read(buf+pos, len-pos)) + if count == 0 then error('short read') end + pos = pos + count + end + return ffi.string(buf, len) +end + +local function serialize_config(config, schema_name) + -- FFS + local schema = yang.load_schema_by_name(schema_name) + local grammar = data.data_grammar_from_schema(schema) + local printer = data.data_string_printer_from_grammar(grammar) + return printer(config) +end + +function run(args) + local schema_name, revision_date, instance_id, config_file = parse_args(args) + local caller = rpc.prepare_caller('snabb-config-leader-v1') + local config = yang.load_configuration(config_file, {schema_name=schema_name}) + local config_str = serialize_config(config, schema_name) + local data = { schema = schema_name, revision = revision_date, + config = config_str } + local msg, parse = rpc.prepare_call(caller, 'load-config', data) + local socket = assert(S.socket("unix", "stream")) + local tail = instance_id..'/config-leader-socket' + local by_name = S.t.sockaddr_un(shm.root..'/by-name/'..tail) + local by_pid = S.t.sockaddr_un(shm.root..'/'..tail) + if not socket:connect(by_name) and not socket:connect(by_pid) then + io.stderr:write( + "Could not connect to config leader socket on Snabb instance '".. + instance_id.."'.\n") + main.exit(1) + end + socket:write(tostring(#msg)..'\n'..msg) + local len = read_length(socket) + msg = read_msg(socket, len) + socket:close() + main.exit(0) +end From 80b4a35166e385275edbce7d28e4b1b7ab9d263e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 16 Nov 2016 09:49:33 +0100 Subject: [PATCH 238/631] Rename action_queue to action_codec --- src/apps/config/{action_queue.lua => action_codec.lua} | 10 +++++----- src/apps/config/follower.lua | 4 ++-- src/apps/config/leader.lua | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) rename src/apps/config/{action_queue.lua => action_codec.lua} (97%) diff --git a/src/apps/config/action_queue.lua b/src/apps/config/action_codec.lua similarity index 97% rename from src/apps/config/action_queue.lua rename to src/apps/config/action_codec.lua index 92f50118b5..af8778e56a 100644 --- a/src/apps/config/action_queue.lua +++ b/src/apps/config/action_codec.lua @@ -141,7 +141,7 @@ local function encoder() return encoder end -function encode_action(action) +function encode(action) local name, args = unpack(action) local codec = encoder() codec:uint32(assert(action_codes[name], name)) @@ -177,14 +177,14 @@ local function decoder(buf, len) return decoder end -function decode_action(buf, len) +function decode(buf, len) local codec = decoder(buf, len) local name = assert(action_names[codec:uint32()]) return { name, assert(actions[name], name)(codec) } end function selftest () - print('selftest: apps.config.action_queue') + print('selftest: apps.config.action_codec') local function serialize(data) local tmp = random_file_name() print('serializing to:', tmp) @@ -200,8 +200,8 @@ function selftest () serialize(1) serialize(1LL) local function test_action(action) - local encoded, len = encode_action(action) - local decoded = decode_action(encoded, len) + local encoded, len = encode(action) + local decoded = decode(encoded, len) assert(lib.equal(action, decoded)) end local appname, linkname, linkspec = 'foo', 'bar', 'foo.a -> bar.q' diff --git a/src/apps/config/follower.lua b/src/apps/config/follower.lua index 4876d0b9a9..42eb3f52ad 100644 --- a/src/apps/config/follower.lua +++ b/src/apps/config/follower.lua @@ -10,7 +10,7 @@ local app = require("core.app") local shm = require("core.shm") local app_graph = require("core.config") local channel = require("apps.config.channel") -local action_queue = require("apps.config.action_queue") +local action_codec = require("apps.config.action_codec") Follower = { config = { @@ -32,7 +32,7 @@ function Follower:handle_actions_from_leader() while true do local buf, len = channel:peek_message() if not buf then break end - local action = action_queue.decode_action(buf, len) + local action = action_codec.decode(buf, len) app.apply_config_actions({action}) channel:discard_message(len) if action[1] == 'start_app' or action[1] == 'reconfig_app' then diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 3776594c95..9c56c40429 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -10,7 +10,7 @@ local rpc = require("lib.yang.rpc") local app = require("core.app") local shm = require("core.shm") local app_graph = require("core.config") -local action_queue = require("apps.config.action_queue") +local action_codec = require("apps.config.action_codec") local channel = require("apps.config.channel") Leader = { @@ -79,7 +79,7 @@ end function Leader:enqueue_config_actions (actions) local messages = {} for _,action in ipairs(actions) do - local buf, len = action_queue.encode_action(action) + local buf, len = action_codec.encode(action) table.insert(messages, { buf=buf, len=len }) end for _,follower in ipairs(self.followers) do From b354d1a94707e03193c587fba21efa556b18fe56 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 16 Nov 2016 10:47:04 +0100 Subject: [PATCH 239/631] Add apps.config documentation --- src/apps/config/README.md | 145 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 src/apps/config/README.md diff --git a/src/apps/config/README.md b/src/apps/config/README.md new file mode 100644 index 0000000000..bdf976cb20 --- /dev/null +++ b/src/apps/config/README.md @@ -0,0 +1,145 @@ +# Config leader and follower + +Sometimes you want to query the state or configuration of a running +Snabb data plane, or reload its configuration, or incrementally update +that configuration. However, you want to minimize the impact of +configuration query and update on data plane performance. The +`Leader` and `Follower` apps are here to fulfill this need, while +minimizing performance overhead. + +The high-level design is that a `Leader` app is responsible for +knowing the state and configuration of a data plane. The leader +offers an interface to allow the outside world to query the +configuration and state, and to request configuration updates. To +avoid data-plane overhead, the `Leader` app should be deployed in a +separate process. Because it knows the data-plane state, it can +respond to queries directly, without involving the data plane. It +processes update requests into a form that the data plane can handle, +and feeds those requests to the data plane via a high-performance +back-channel. + +The data plane runs a `Follower` app that reads and applies update +messages sent to it from the leader. Checking for update availability +requires just a memory access, not a system call, so the overhead of +including a follower in the data plane is very low. + +## Two protocols + +The leader communicates with its followers using a private protocol. +Because the leader and the follower are from the same Snabb version, +the details of this protocol are subject to change. The private +protocol's only design constraint is that it should cause the lowest +overhead for the data plane. + +The leader communicates with the world via a public protocol. The +"snabb config" command-line tool speaks this protocol. "snabb config +get foo /bar" will find the local Snabb instance named "foo", open the +UNIX socket that the "foo" instance is listening on, issue a request, +then read the response, then close the socket. + +## Public protocol + +The design constraint on the public protocol is that it be expressive +and future-proof. We also want to enable the leader to talk to more +than one "snabb config" at a time. In particular someone should be +able to have a long-lived "snabb config listen" session open, and that +shouldn't impede someone else from doing a "snabb config get" to read +state. + +To this end the public protocol container is very simple: + +``` +Message = Length "\n" RPC* +``` + +Length is a base-10 string of characters indicating the length of the +message. There may be a maximum length restriction. This requires +that "snabb config" build up the whole message as a string and measure +its length, but that's OK. Knowing the length ahead of time allows +"snabb config" to use nonblocking operations to slurp up the whole +message as a string. A partial read can be resumed later. The +message can then be parsed without fear of blocking the main process. + +The RPC is an RPC request or response for the +[`snabb-config-leader-v1` YANG +schema](../../lib/yang/snabb-config-leader-v1.yang), expressed in the +Snabb [textual data format for YANG data](../../lib/yang/README.md). +For example the `snabb-config-leader-v1` schema supports a +`get-config` RPC defined like this in the schema: + +``` +rpc get-config { + input { + leaf schema { type string; mandatory true; } + leaf revision { type string; } + leaf path { type string; default "/"; } + } + output { + leaf config { type string; } + } +} +``` + +A request to this RPC might look like: + +``` +get-config { + schema snabb-softwire-v1; + path "/foo"; +} +``` + +As you can see, non-mandatory inputs can be left out. A response +might look like: + +``` +get-config { + config "blah blah blah"; +} +``` + +Responses are prefixed by the RPC name. One message can include a +number of RPCs; the RPCs will be made in order. See the +[`snabb-config-leader-v1` YANG +schema](../../lib/yang/snabb-config-leader-v1.yang) for full details +of available RPCs. + +## Private protocol + +The leader maintains a configuration for the program as a whole. As +it gets requests, it computes the set of changes to app graphs that +would be needed to apply that configuration. These changes are then +passed through the private protocol to the follower. No response from +the follower is necessary. + +In some remote or perhaps not so remote future, all Snabb apps will +have associated YANG schemas describing their individual +configurations. In this happy future, the generic way to ship +configurations from the leader to a follower is by the binary +serialization of YANG data, implemented already in the YANG modules. +Until then however, there is also generic Lua data without a schema. +The private protocol supports both kinds of information transfer. + +In the meantime, the way to indicate that an app's configuration data +conforms to a YANG schema is to set the `schema_name` property on the +app's class. + +The private protocol consists of binary messages passed over a ring +buffer. A follower's leader writes to the buffer, and the follower +reads from it. There are no other readers or writers. Given that a +message may in general be unbounded in size, whereas a ring buffer is +naturally fixed, messages which may include arbtrary-sized data may be +forced to put that data in the filesystem, and refer to it from the +messages in the ring buffer. Since this file system is backed by +`tmpfs`, stalls will be minimal. + +## User interface + +The above sections document how the leader and follower apps are +implemented so that a data-plane developer can understand the overhead +of run-time (re)configuration. End users won't be typing at a UNIX +socket though; we include the `snabb config` program as a command-line +interface to this functionality. + +See [the `snabb config` documentation](../../program/config/README.md) +for full details. From 9256bf8082e645b8818683769e17e71992027f85 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 16 Nov 2016 12:49:58 +0100 Subject: [PATCH 240/631] Add "snabb config" README It's a bit aspirational at this point though. --- src/program/config/README.md | 258 +++++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 src/program/config/README.md diff --git a/src/program/config/README.md b/src/program/config/README.md new file mode 100644 index 0000000000..2288c7ee9d --- /dev/null +++ b/src/program/config/README.md @@ -0,0 +1,258 @@ +# Data-plane configuration + +Would be nice if you could update a Snabb program's configuration +while it's running, wouldn't it? Well never fear, `snabb config` is +here. Provided the data-plane author enabled this feature on their +side, users can run `snabb config` commands to query state or +configuration, provide a new configuration, or incrementally update +the existing configuration of a Snabb instance. + +## `snabb config` + +`snabb config` is a family of Snabb commands. Its sub-commands +include: + +* [`snabb config get`](./get/README.md): read configuration data + +* [`snabb config get-state`](./get_state/README.md): read state data + +* [`snabb config load`](./load/README.md): load a new configuration + +* [`snabb config set`](./set/README.md): incrementally update configuration + +* [`snabb config add`](./add/README.md): augment configuration, for + example by adding a routing table entry + +* [`snabb config delete`](./delete/README.md): remove a component from + a configuration, for example removing a routing table entry + +* [`snabb config listen`](./listen/README.md): provide an interface to + the `snabb config` functionality over a persistent socket, to + minimize per-operation cost + +The `snabb config get` commands are the normal way that Snabb users +interact with Snabb applications in an ad-hoc fashion via the command +line. `snabb config listen` is the standard way that a NETCONF agent +like Sysrepo interacts with a Snabb network function. + +### Configuration model + +Most `snabb config` commands are invoked in a uniform way: + +``` +snabb config SUBCOMMAND -m MODULE ID PATH [VALUE] +``` + +`snabb config` speaks a data model that is based on YANG, for minimum +impedance mismatch between NETCONF agents and Snabb applications. The +`-m MODULE` option allows the caller to indicate the YANG module that +they want to use, and for the purposes of the lwAFTR might be `-m +ietf-softwire`. + +`ID` identifies the particular Snabb instance to talk to, and can be a +PID or a name. Snabb supports the ability for an instance to acquire +a name, which is often more convenient than dealing with changing +PIDs. + +`PATH` specifies a subset of the configuration tree to operate on. + +Let's imagine that the configuration for the Snabb instance in +question is modelled by the following YANG schema: + +``` +module snabb-simple-router { + namespace snabb:simple-router; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + leaf active { type boolean; default true; } + + container routes { + presence true; + list route { + key addr; + leaf addr { type inet:ipv4-address; mandatory true; } + leaf port { type uint8 { range 0..11; }; mandatory true; } + } + } +} +``` + +In this case then, we would pass `-m snabb-simple-router` to all of +our `snabb config` invocations that talk to this router. + +The configuration for a Snabb instance can be expressed in a text +format that is derived from the schema. In this case it could look +like: + +``` +active true; +routes { + route { addr 1.2.3.4; port 1; } + route { addr 2.3.4.5; port 2; } +} +``` + +The surface syntax of data is the same as for YANG schemas; you can +have end-of-line comments with `//`, larger comments with `/* ... */`, +and the YANG schema quoting rules for strings apply. + +Note that containers like `route {}` only appear in the data syntax if +they are marked as `presence true;` in the schema. + +So indeed, `snabb config get -m snabb-simple-router ID /` +might print out just the output given above. + +Users can limit their query to a particular subtree via passing a +different `PATH`. For example, with the same configuration, we can +query just the `active` value: + +``` +$ snabb config get -m snabb-simple-router ID /active +true +``` + +`PATH` is in a subset of XPath, which should be familiar to NETCONF +operators. Note that the XPath selector operates over the data, not +the schema, so the path components should reflect the data. + +A `list` is how YANG represents associations between keys and values. +To query an element of a `list` item, use an XPath selector; for +example, to get the entry for IPv4 `1.2.3.4`, do: + +``` +$ snabb config get -m snabb-simple-router ID /routes/route[addr=1.2.3.4] +route { + addr 1.2.3.4; + port 1; +} +``` + +Or to just get the port: + +``` +$ snabb config get -m snabb-simple-router ID /routes/route[addr=1.2.3.4]/port +1 +``` + +Likewise, to change the port for `1.2.3.4`, do: + +``` +$ snabb config set -m snabb-simple-router ID /routes/route[addr=1.2.3.4]/port 7 +``` + +Values can be large, so it's also possible to take them from `stdin`. +Do this by omitting the value: + +``` +$ cat /tmp/my-configuration | snabb config set -m snabb-simple-router ID / +``` + +Resetting the whole configuration is such a common operation that it +has a special command that takes a file name instead of a path: + +``` +$ snabb config load -m snabb-simple-router ID /tmp/my-configuration +``` + +Using `snabb config load` has the advantage that any configuration +error has a corresponding source location. + +`snabb config` can also delete part of a configuration, but only on +configuration that corresponds to YANG schema `leaf` or `leaf-list` +nodes: + +``` +$ snabb config delete -m snabb-simple-router ID /routes/route[addr=1.2.3.4] +``` + +One can of course augment a configuration as well: + +``` +$ snabb config add -m snabb-simple-router ID /routes +route { + addr 4.5.6.7; + port 11; +} +``` + +### Machine interface + +The `listen` interface will support all of these operations with a +simple JSON protocol. Each request will be one JSON object with the +following properties: + +- `id`: A request identifier; a string. Not used by `snabb config + listen`; just a convenience for the other side. + +- `verb`: The action to perform; one of `get-state`, `get`, `set`, + `add`, or `delete`. A string. + +- `path`: A path identifying the configuration or state data on which + to operate. A string. + +- `value`: Only present for the set and add verbs, a string + representation of the YANG instance data to set or add. The value is + encoded as a string in the same syntax that the `snabb config set` + accepts. + +Each response from the server will also be one JSON object, with the following properties: + +- `id`: The identifier corresponding to the request. A string. + +- `status`: Either ok or error. A string. + +- `value`: If the request was a `get` request and the status is `ok`, + then the `value` property is present, containing a `Data` + representation of the value. A string. + +Error messages may have additional properties which can help diagnose +the reason for the error. These properties will be defined in the +future. + +``` +$ snabb config listen -m snabb-simple-router ID +{ "id": "0", "verb": "get", "path": "/routes/route[addr=1.2.3.4]/port" } +{ "id": "1", "verb": "get", "path": "/routes/route[addr=2.3.4.5]/port" } +{ "id": "0", "status": "ok", "value: "1" } +{ "id": "1}, "status": "ok", "value: "2" } +``` + +The above transcript indicates that requests may be pipelined: the +client to `snabb config` listen may make multiple requests without +waiting for responses. (For clarity, the first two JSON objects in the +above transcript were entered by the user, in the console in this +case; the second two are printed out by `snabb config` in response.) + +The `snabb config listen` program will acquire exclusive write access +to the data plane, preventing other `snabb config` invocations from +modifying the configuration. In this way there is no need to provide +for notifications of changes made by other configuration clients. + +### Multiple schemas + +Support is planned for multiple schemas. For example the Snabb lwAFTR +uses a native YANG schema to model its configuration and state, but +operators would also like to interact with the lwAFTR using the +relevant standardized schemas. Work here is ongoing. + +## How does it work? + +The Snabb instance itself should be running in *multi-process mode*, +whereby there is one main process that shepherds a number of worker +processes. The workers perform the actual data-plane functionality, +are typically bound to reserved CPU and NUMA nodes, and have +soft-real-time constraints. The main process however doesn't have +much to do; it just coordinates the workers. + +The main process runs a special app in its engine that listens on a +UNIX socket for special remote procedure calls, translates those calls +to updates that the data plane should apply, and dispatches those +updates to the data plane in an efficient way. See the [`apps.config` +documentation](../../apps/config/README.md) for full details. + +Some data planes, like the lwAFTR, add hooks to the `set`, `add`, and +`delete` subcommands of `snabb config` to allow even more efficient +incremental updates, for example updating the binding table in place +via a custom protocol. From cbca5b97c6b1ebc6cd8ac1eb80132d241a402dd0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 16 Nov 2016 13:07:41 +0100 Subject: [PATCH 241/631] Call link methods Sadly this makes applying a configuration N^2 in the number of links that an app has. This could be improved, but maybe it doesn't matter given that so few apps have link methods. --- src/core/app.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/app.lua b/src/core/app.lua index f2f24d4b26..3ba27bb1ff 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -214,12 +214,14 @@ function apply_config_actions (actions) local link = app.output[linkname] app.output[linkname] = nil remove_link_from_array(app.output, link) + if app.link then app:link() end end function ops.unlink_input (appname, linkname) local app = app_table[appname] local link = app.input[linkname] app.input[linkname] = nil remove_link_from_array(app.input, link) + if app.link then app:link() end end function ops.free_link (linkspec) link.free(link_table[linkspec], linkspec) @@ -235,12 +237,14 @@ function apply_config_actions (actions) local link = assert(link_table[linkspec]) app.output[linkname] = link table.insert(app.output, link) + if app.link then app:link() end end function ops.link_input (appname, linkname, linkspec) local app = app_table[appname] local link = assert(link_table[linkspec]) app.input[linkname] = link table.insert(app.input, link) + if app.link then app:link() end end function ops.stop_app (name) local app = app_table[name] From 987dd77b61f3bc2f856f6fb8f444e3ddab7d3477 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Wed, 16 Nov 2016 13:15:36 +0000 Subject: [PATCH 242/631] core.memory: Retry on hugetlb allocation errors Fix a regression where the memory module would not retry hugetlb allocations. pcall() was needed to trap allocation errors instead of letting them propagate. The old allocation primitive was written in C and always return NULL on error but the new one is written in Lua and uses assertions. --- src/core/memory.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/memory.lua b/src/core/memory.lua index d62a407d23..da7b9291a1 100644 --- a/src/core/memory.lua +++ b/src/core/memory.lua @@ -60,9 +60,9 @@ function allocate_hugetlb_chunk () local fd, err = syscall.open("/proc/sys/vm/nr_hugepages","rdonly") assert(fd, tostring(err)) fd:flock("ex") - for i =1, 3 do - local page = allocate_huge_page(huge_page_size, true) - if page ~= nil then + for i = 1, 3 do + local ok, page = pcall(allocate_huge_page, huge_page_size, true) + if ok then fd:flock("un") fd:close() return page From d7194ad05f0a6d6bb8dbceb71b230c08522f7a1f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 16 Nov 2016 14:20:28 +0100 Subject: [PATCH 243/631] Always unlink endpoints when links go away --- src/core/app.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 3ba27bb1ff..6bfedab3e6 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -137,9 +137,9 @@ function compute_config_actions (old, new) for linkspec in pairs(old.links) do if not new.links[linkspec] then local fa, fl, ta, tl = config.parse_link(linkspec) - if old.apps[fa] then add_action('unlink_output', fa, fl) end - if old.apps[ta] then add_action('unlink_input', ta, tl) end - if not new[linkspec] then add_action('free_link', linkspec) end + add_action('unlink_output', fa, fl) + add_action('unlink_input', ta, tl) + add_action('free_link', linkspec) end end From b551d273a4724377360acf979567e7b1133e7151 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 16 Nov 2016 16:52:07 +0100 Subject: [PATCH 244/631] Fix perf hazard in restart_dead_apps The local function in the cold path of restart_dead_apps introduced a perf hazard, making LuaJIT add a UCLO instruction to the early return path, which is a NYI bytecode that forced frequent trace aborts. This showed up as about a 25% perf penalty on "snabb snabbmark basic1 100e6". --- src/core/app.lua | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 6bfedab3e6..373e473c5f 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -95,19 +95,19 @@ function restart_dead_apps () for name, app in pairs(app_table) do if app.dead and (now() - app.dead.time) >= restart_delay then - local function add_action(action, ...) - table.insert(actions, { action, { ... } }) - end io.stderr:write(("Restarting %s (died at %f: %s)\n") :format(name, app.dead.time, app.dead.error)) - add_action('stop_app', name) - add_action('start_app', name, - configuration.apps[name].class, - configuration.apps[name].arg) + local info = configuration.apps[name] + table.insert(actions, {'stop_app', {name}}) + table.insert(actions, {'start_app', {name, info.class, info.arg}}) for linkspec in pairs(configuration.links) do local fa, fl, ta, tl = config.parse_link(linkspec) - if fa == name then add_action('link_output', fa, fl, linkspec) end - if ta == name then add_action('link_input', ta, tl, linkspec) end + if fa == name then + table.insert(actions, {'link_output', {fa, fl, linkspec}}) + end + if ta == name then + table.insert(actions, {'link_input', {ta, tl, linkspec}}) + end end end end From c708b62d2455b9d979cacc794a2314d6b27b7d34 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 16 Nov 2016 17:06:03 +0100 Subject: [PATCH 245/631] Simplification in compute_config_actions This simplification removes a varargs-using local function, which might be a performance hazard. It's a neutral change code-wise but I am pushing it to see what the bots think of it. --- src/core/app.lua | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 373e473c5f..4b34d64019 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -129,23 +129,22 @@ end -- Return the configuration actions needed to migrate from old config to new. function compute_config_actions (old, new) local actions = {} - local function add_action(action, ...) - table.insert(actions, { action, { ... } }) - end -- First determine the links that are going away and remove them. for linkspec in pairs(old.links) do if not new.links[linkspec] then local fa, fl, ta, tl = config.parse_link(linkspec) - add_action('unlink_output', fa, fl) - add_action('unlink_input', ta, tl) - add_action('free_link', linkspec) + table.insert(actions, {'unlink_output', {fa, fl}}) + table.insert(actions, {'unlink_input', {ta, tl}}) + table.insert(actions, {'free_link', {linkspec}}) end end -- Do the same for apps. for appname, info in pairs(old.apps) do - if not new.apps[appname] then add_action('stop_app', appname) end + if not new.apps[appname] then + table.insert(actions, {'stop_app', {appname}}) + end end -- Start new apps, restart reclassed apps, or reconfigure apps with @@ -154,18 +153,18 @@ function compute_config_actions (old, new) for appname, info in pairs(new.apps) do local class, arg = info.class, info.arg if not old.apps[appname] then - add_action('start_app', appname, info.class, info.arg) + table.insert(actions, {'start_app', {appname, class, arg}}) fresh_apps[appname] = true elseif old.apps[appname].class ~= class then - add_action('stop_app', appname) - add_action('start_app', appname, info.class, info.arg) + table.insert(actions, {'stop_app', {appname}}) + table.insert(actions, {'start_app', {appname, class, arg}}) fresh_apps[appname] = true elseif not lib.equal(old.apps[appname].arg, arg) then if class.reconfig then - add_action('reconfig_app', appname, info.arg) + table.insert(actions, {'reconfig_app', {appname, arg}}) else - add_action('stop_app', appname) - add_action('start_app', appname, info.class, info.arg) + table.insert(actions, {'stop_app', {appname}}) + table.insert(actions, {'start_app', {appname, class, arg}}) fresh_apps[appname] = true end else @@ -175,17 +174,17 @@ function compute_config_actions (old, new) end -- Now rebuild links. - for linkspec in pairs(new.links) do + for linkspec,_ in pairs(new.links) do local fa, fl, ta, tl = config.parse_link(linkspec) local fresh_link = not old.links[linkspec] - if fresh_link then add_action('new_link', linkspec) end + if fresh_link then table.insert(actions, {'new_link', {linkspec}}) end if not new.apps[fa] then error("no such app: " .. fa) end if not new.apps[ta] then error("no such app: " .. ta) end if fresh_link or fresh_apps[fa] then - add_action('link_output', fa, fl, linkspec) + table.insert(actions, {'link_output', {fa, fl, linkspec}}) end if fresh_link or fresh_apps[ta] then - add_action('link_input', ta, tl, linkspec) + table.insert(actions, {'link_input', {ta, tl, linkspec}}) end end From 4242854a70906a32d5b21bc894118b833d01cde5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 10:01:41 +0100 Subject: [PATCH 246/631] Fix perf hazard in leader socket accept() Although the leader does not have real-time requirements, in this meantime in which we don't have a multiprocess world yet we are running it in the data plane, so we need to ensure that it doesn't cause trace aborts. Creating a new sockaddr each time was causing trace aborts, so here we fix it. Anyway improving this will make traces cleaner. --- src/apps/config/leader.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 9c56c40429..7cbced3751 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -111,11 +111,12 @@ function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end +local dummy_unix_sockaddr = S.t.sockaddr_un() + function Leader:handle_calls_from_peers() local peers = self.peers while true do - local sa = S.t.sockaddr_un() - local fd, err = self.socket:accept(sa) + local fd, err = self.socket:accept(dummy_unix_sockaddr) if not fd then if err.AGAIN then break end assert(nil, err) From 50f644180451c59437ae601d67e06b8e34edd4e8 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 10:29:58 +0100 Subject: [PATCH 247/631] Rename "snabb config load-config" to "snabb config load" Also fix error message when load has no arguments. --- src/program/config/config.lua | 2 +- src/program/config/{load_config => load}/README.inc | 2 +- .../config/{load_config/load_config.lua => load/load.lua} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename src/program/config/{load_config => load}/README.inc (78%) rename src/program/config/{load_config/load_config.lua => load/load.lua} (96%) diff --git a/src/program/config/config.lua b/src/program/config/config.lua index 6acd10fad1..134118611d 100644 --- a/src/program/config/config.lua +++ b/src/program/config/config.lua @@ -12,7 +12,7 @@ local function parse_args(args) local handlers = {} handlers.h = function() show_usage(0) end args = lib.dogetopt(args, handlers, "h", {help="h"}) - if #args <= 1 then show_usage(1) end + if #args < 1 then show_usage(1) end return args end diff --git a/src/program/config/load_config/README.inc b/src/program/config/load/README.inc similarity index 78% rename from src/program/config/load_config/README.inc rename to src/program/config/load/README.inc index f5e7c99d79..789078554f 100644 --- a/src/program/config/load_config/README.inc +++ b/src/program/config/load/README.inc @@ -1,5 +1,5 @@ Usage: - snabb config load-config [OPTIONS] ID FILE + snabb config load [OPTIONS] ID FILE Available options: -s SCHEMA diff --git a/src/program/config/load_config/load_config.lua b/src/program/config/load/load.lua similarity index 96% rename from src/program/config/load_config/load_config.lua rename to src/program/config/load/load.lua index 5aeedba70f..f3b8113702 100644 --- a/src/program/config/load_config/load_config.lua +++ b/src/program/config/load/load.lua @@ -13,7 +13,7 @@ local data = require("lib.yang.data") local tab_spaces = 2 local function show_usage(status) - print(require("program.config.load_config.README_inc")) + print(require("program.config.load.README_inc")) main.exit(status) end @@ -23,7 +23,7 @@ local function parse_args(args) function handlers.h() show_usage(0) end function handlers.s(arg) schema_name = arg end function handlers.r(arg) revision_date = arg end - args = lib.dogetopt(args, handlers, "h:s:r", + args = lib.dogetopt(args, handlers, "hs:r:", {help="h", ['schema-name']="s", schema="s", ['revision-date']="r", revision="r"}) if not schema_name then show_usage(1) end From c43e98f2e21d48759450067dc0cb35f6572d18a5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 10:32:33 +0100 Subject: [PATCH 248/631] Rename "snabb config get-config" to "snabb config get" --- src/program/config/{get_config => get}/README.inc | 2 +- src/program/config/{get_config/get_config.lua => get/get.lua} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/program/config/{get_config => get}/README.inc (78%) rename src/program/config/{get_config/get_config.lua => get/get.lua} (95%) diff --git a/src/program/config/get_config/README.inc b/src/program/config/get/README.inc similarity index 78% rename from src/program/config/get_config/README.inc rename to src/program/config/get/README.inc index 83bbfce3d8..208310727e 100644 --- a/src/program/config/get_config/README.inc +++ b/src/program/config/get/README.inc @@ -1,5 +1,5 @@ Usage: - snabb config get-config [OPTIONS] ID PATH + snabb config get [OPTIONS] ID PATH Available options: -s SCHEMA diff --git a/src/program/config/get_config/get_config.lua b/src/program/config/get/get.lua similarity index 95% rename from src/program/config/get_config/get_config.lua rename to src/program/config/get/get.lua index 884e4ebd60..d451eb3a26 100644 --- a/src/program/config/get_config/get_config.lua +++ b/src/program/config/get/get.lua @@ -11,7 +11,7 @@ local rpc = require("lib.yang.rpc") local tab_spaces = 2 local function show_usage(status) - print(require("program.config.get_config.README_inc")) + print(require("program.config.get.README_inc")) main.exit(status) end @@ -21,7 +21,7 @@ local function parse_args(args) function handlers.h() show_usage(0) end function handlers.s(arg) schema_name = arg end function handlers.r(arg) revision_date = arg end - args = lib.dogetopt(args, handlers, "h:s:r", + args = lib.dogetopt(args, handlers, "hs:r:", {help="h", ['schema-name']="s", schema="s", ['revision-date']="r", revision="r"}) if not schema_name then show_usage(1) end From 87c887cf868ba4bce3b7c9c016058262710deb03 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 11:40:06 +0100 Subject: [PATCH 249/631] Refactor "snabb config" More functionality moves to a new "common.lua" file. --- src/program/config/common.lua | 106 +++++++++++++++++++++++++++++++++ src/program/config/get/get.lua | 69 +++------------------ 2 files changed, 113 insertions(+), 62 deletions(-) create mode 100644 src/program/config/common.lua diff --git a/src/program/config/common.lua b/src/program/config/common.lua new file mode 100644 index 0000000000..ab652fa1dd --- /dev/null +++ b/src/program/config/common.lua @@ -0,0 +1,106 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local S = require("syscall") +local ffi = require("ffi") +local lib = require("core.lib") +local shm = require("core.shm") +local rpc = require("lib.yang.rpc") + +function show_usage(command, status, err_msg) + if err_msg then print('error: '..err_msg) end + print(require("program.config."..command:gsub('-','_')..".README_inc")) + main.exit(status) +end + +local parse_command_line_opts = { + command = { required=true }, + with_config_file = { default=false }, + with_path = { default=false }, + with_value = { default=false } +} + +function parse_command_line(args, opts) + opts = lib.parse(opts, parse_command_line_opts) + local function err(msg) show_usage(opts.command, 1, msg) end + local schema_name, revision_date + local handlers = {} + function handlers.h() show_usage(opts.command, 0) end + function handlers.s(arg) schema_name = arg end + function handlers.r(arg) revision_date = arg end + args = lib.dogetopt(args, handlers, "hs:r:", + {help="h", ['schema-name']="s", schema="s", + ['revision-date']="r", revision="r"}) + if not schema_name then err("missing --schema arg") end + if #args == 0 then err() end + local instance_id = table.remove(args, 1) + local ret = { schema_name, revision_date, instance_id } + if opts.with_config_file then + if #args == 0 then err("missing config file argument") end + local file = table.remove(args, 1) + table.insert(ret, yang.load_configuration(file, schema_name)) + end + if opts.with_path then + if #args == 0 then err("missing path argument") end + local path = table.remove(args, 1) + -- Waiting on our XPath parsing library :) + if path ~= '/' then err("paths other than / currently unimplemented") end + table.insert(ret, path) + end + if opts.with_value then + local parser = data_parser(schema_name, path) + if #args == 0 then + table.insert(ret, parser(io.stdin:read('*a'))) + else + table.insert(ret, parser(table.remove(args, 1))) + end + end + if #args ~= 0 then err("too many arguments") end + return unpack(ret) +end + +function open_socket_or_die(instance_id) + S.signal('pipe', 'ign') + local socket = assert(S.socket("unix", "stream")) + local tail = instance_id..'/config-leader-socket' + local by_name = S.t.sockaddr_un(shm.root..'/by-name/'..tail) + local by_pid = S.t.sockaddr_un(shm.root..'/'..tail) + if not socket:connect(by_name) and not socket:connect(by_pid) then + io.stderr:write( + "Could not connect to config leader socket on Snabb instance '".. + instance_id.."'.\n") + main.exit(1) + end + return socket +end + +function send_message(socket, msg_str) + socket:write(tostring(#msg_str)..'\n'..msg_str) +end + +local function read_length(socket) + local len = 0 + while true do + local ch = assert(socket:read(nil, 1)) + assert(ch ~= '', 'short read') + if ch == '\n' then return len end + assert(tonumber(ch), 'not a number: '..ch) + len = len * 10 + tonumber(ch) + assert(len < 1e8, 'length too long: '..len) + end +end + +local function read_msg(socket, len) + local buf = ffi.new('uint8_t[?]', len) + local pos = 0 + while pos < len do + local count = assert(socket:read(buf+pos, len-pos)) + if count == 0 then error('short read') end + pos = pos + count + end + return ffi.string(buf, len) +end + +function recv_message(socket) + return read_msg(socket, read_length(socket)) +end diff --git a/src/program/config/get/get.lua b/src/program/config/get/get.lua index d451eb3a26..35d778ddf2 100644 --- a/src/program/config/get/get.lua +++ b/src/program/config/get/get.lua @@ -3,75 +3,20 @@ module(..., package.seeall) local S = require("syscall") local ffi = require("ffi") -local lib = require("core.lib") local shm = require("core.shm") local rpc = require("lib.yang.rpc") - --- Number of spaces a tab should consist of when indenting config. -local tab_spaces = 2 - -local function show_usage(status) - print(require("program.config.get.README_inc")) - main.exit(status) -end - -local function parse_args(args) - local schema_name, revision_date - local handlers = {} - function handlers.h() show_usage(0) end - function handlers.s(arg) schema_name = arg end - function handlers.r(arg) revision_date = arg end - args = lib.dogetopt(args, handlers, "hs:r:", - {help="h", ['schema-name']="s", schema="s", - ['revision-date']="r", revision="r"}) - if not schema_name then show_usage(1) end - if #args < 2 or #args > 2 then show_usage(1) end - local instance_id, path = unpack(args) - return schema_name, revision_date, instance_id, path -end - -local function read_length(socket) - local len = 0 - while true do - local ch = assert(socket:read(nil, 1)) - assert(ch ~= '', 'short read') - if ch == '\n' then return len end - assert(tonumber(ch), 'not a number: '..ch) - len = len * 10 + tonumber(ch) - assert(len < 1e8, 'length too long: '..len) - end -end - -local function read_msg(socket, len) - local buf = ffi.new('uint8_t[?]', len) - local pos = 0 - while pos < len do - local count = assert(socket:read(buf+pos, len-pos)) - if count == 0 then error('short read') end - pos = pos + count - end - return ffi.string(buf, len) -end +local common = require("program.config.common") function run(args) - local schema_name, revision_date, instance_id, path = parse_args(args) + local schema_name, revision_date, instance_id, path = + common.parse_command_line(args, { command='get', with_path=true }) local caller = rpc.prepare_caller('snabb-config-leader-v1') local data = { schema = schema_name, revision = revision_date, path = path } local msg, parse = rpc.prepare_call(caller, 'get-config', data) - local socket = assert(S.socket("unix", "stream")) - local tail = instance_id..'/config-leader-socket' - local by_name = S.t.sockaddr_un(shm.root..'/by-name/'..tail) - local by_pid = S.t.sockaddr_un(shm.root..'/'..tail) - if not socket:connect(by_name) and not socket:connect(by_pid) then - io.stderr:write( - "Could not connect to config leader socket on Snabb instance '".. - instance_id.."'.\n") - main.exit(1) - end - socket:write(tostring(#msg)..'\n'..msg) - local len = read_length(socket) - msg = read_msg(socket, len) + local socket = common.open_socket_or_die(instance_id) + common.send_message(socket, msg) + local reply = common.recv_message(socket) socket:close() - print(parse(msg).config) + print(parse(reply).config) main.exit(0) end From 346ae474bedccbda3c0b1453bd100e5c888caef6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 11:54:19 +0100 Subject: [PATCH 250/631] Refactor "snabb config load" Taking advantage of common.lua, and moving more things there. --- src/program/config/common.lua | 14 +++++- src/program/config/get/get.lua | 3 -- src/program/config/load/load.lua | 85 ++++---------------------------- 3 files changed, 22 insertions(+), 80 deletions(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index ab652fa1dd..47ceb27d8a 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -6,6 +6,8 @@ local ffi = require("ffi") local lib = require("core.lib") local shm = require("core.shm") local rpc = require("lib.yang.rpc") +local yang = require("lib.yang.yang") +local data = require("lib.yang.data") function show_usage(command, status, err_msg) if err_msg then print('error: '..err_msg) end @@ -38,7 +40,8 @@ function parse_command_line(args, opts) if opts.with_config_file then if #args == 0 then err("missing config file argument") end local file = table.remove(args, 1) - table.insert(ret, yang.load_configuration(file, schema_name)) + local opts = {schema_name=schema_name, revision_date=revision_date} + table.insert(ret, yang.load_configuration(file, opts)) end if opts.with_path then if #args == 0 then err("missing path argument") end @@ -74,6 +77,15 @@ function open_socket_or_die(instance_id) return socket end +function serialize_config(config, schema_name, path) + assert(path == nil or path == "/") + -- FFS + local schema = yang.load_schema_by_name(schema_name) + local grammar = data.data_grammar_from_schema(schema) + local printer = data.data_string_printer_from_grammar(grammar) + return printer(config) +end + function send_message(socket, msg_str) socket:write(tostring(#msg_str)..'\n'..msg_str) end diff --git a/src/program/config/get/get.lua b/src/program/config/get/get.lua index 35d778ddf2..8d887630d9 100644 --- a/src/program/config/get/get.lua +++ b/src/program/config/get/get.lua @@ -1,9 +1,6 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -local S = require("syscall") -local ffi = require("ffi") -local shm = require("core.shm") local rpc = require("lib.yang.rpc") local common = require("program.config.common") diff --git a/src/program/config/load/load.lua b/src/program/config/load/load.lua index f3b8113702..bb72913efc 100644 --- a/src/program/config/load/load.lua +++ b/src/program/config/load/load.lua @@ -1,89 +1,22 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -local S = require("syscall") -local ffi = require("ffi") -local lib = require("core.lib") -local shm = require("core.shm") local rpc = require("lib.yang.rpc") -local yang = require("lib.yang.yang") -local data = require("lib.yang.data") - --- Number of spaces a tab should consist of when indenting config. -local tab_spaces = 2 - -local function show_usage(status) - print(require("program.config.load.README_inc")) - main.exit(status) -end - -local function parse_args(args) - local schema_name, revision_date - local handlers = {} - function handlers.h() show_usage(0) end - function handlers.s(arg) schema_name = arg end - function handlers.r(arg) revision_date = arg end - args = lib.dogetopt(args, handlers, "hs:r:", - {help="h", ['schema-name']="s", schema="s", - ['revision-date']="r", revision="r"}) - if not schema_name then show_usage(1) end - if #args < 2 or #args > 2 then show_usage(1) end - local instance_id, config_file = unpack(args) - return schema_name, revision_date, instance_id, config_file -end - -local function read_length(socket) - local len = 0 - while true do - local ch = assert(socket:read(nil, 1)) - assert(ch ~= '', 'short read') - if ch == '\n' then return len end - assert(tonumber(ch), 'not a number: '..ch) - len = len * 10 + tonumber(ch) - assert(len < 1e8, 'length too long: '..len) - end -end - -local function read_msg(socket, len) - local buf = ffi.new('uint8_t[?]', len) - local pos = 0 - while pos < len do - local count = assert(socket:read(buf+pos, len-pos)) - if count == 0 then error('short read') end - pos = pos + count - end - return ffi.string(buf, len) -end - -local function serialize_config(config, schema_name) - -- FFS - local schema = yang.load_schema_by_name(schema_name) - local grammar = data.data_grammar_from_schema(schema) - local printer = data.data_string_printer_from_grammar(grammar) - return printer(config) -end +local common = require("program.config.common") function run(args) - local schema_name, revision_date, instance_id, config_file = parse_args(args) + local schema_name, revision_date, instance_id, config = + common.parse_command_line(args, + { command='load', with_config_file=true }) local caller = rpc.prepare_caller('snabb-config-leader-v1') - local config = yang.load_configuration(config_file, {schema_name=schema_name}) - local config_str = serialize_config(config, schema_name) + local config_str = common.serialize_config(config, schema_name) local data = { schema = schema_name, revision = revision_date, config = config_str } local msg, parse = rpc.prepare_call(caller, 'load-config', data) - local socket = assert(S.socket("unix", "stream")) - local tail = instance_id..'/config-leader-socket' - local by_name = S.t.sockaddr_un(shm.root..'/by-name/'..tail) - local by_pid = S.t.sockaddr_un(shm.root..'/'..tail) - if not socket:connect(by_name) and not socket:connect(by_pid) then - io.stderr:write( - "Could not connect to config leader socket on Snabb instance '".. - instance_id.."'.\n") - main.exit(1) - end - socket:write(tostring(#msg)..'\n'..msg) - local len = read_length(socket) - msg = read_msg(socket, len) + local socket = common.open_socket_or_die(instance_id) + common.send_message(socket, msg) + local reply = common.recv_message(socket) socket:close() + -- The reply is effectively empty. main.exit(0) end From cf5733f84e66a76f8c65f62d74c37ebd7031e147 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 12:02:36 +0100 Subject: [PATCH 251/631] Further "snabb config" refactors --- src/program/config/common.lua | 10 ++++++++++ src/program/config/get/get.lua | 16 ++++++---------- src/program/config/load/load.lua | 15 +++++---------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 47ceb27d8a..c02765d51d 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -116,3 +116,13 @@ end function recv_message(socket) return read_msg(socket, read_length(socket)) end + +function call_leader(instance_id, method, args) + local caller = rpc.prepare_caller('snabb-config-leader-v1') + local socket = open_socket_or_die(instance_id) + local msg, parse_reply = rpc.prepare_call(caller, method, args) + send_message(socket, msg) + local reply = recv_message(socket) + socket:close() + return parse_reply(reply) +end diff --git a/src/program/config/get/get.lua b/src/program/config/get/get.lua index 8d887630d9..b07dd13c25 100644 --- a/src/program/config/get/get.lua +++ b/src/program/config/get/get.lua @@ -1,19 +1,15 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -local rpc = require("lib.yang.rpc") local common = require("program.config.common") function run(args) local schema_name, revision_date, instance_id, path = - common.parse_command_line(args, { command='get', with_path=true }) - local caller = rpc.prepare_caller('snabb-config-leader-v1') - local data = { schema = schema_name, revision = revision_date, path = path } - local msg, parse = rpc.prepare_call(caller, 'get-config', data) - local socket = common.open_socket_or_die(instance_id) - common.send_message(socket, msg) - local reply = common.recv_message(socket) - socket:close() - print(parse(reply).config) + common.parse_command_line(args, { command='load', with_path=true }) + local response = common.call_leader( + instance_id, 'get-config', + { schema = schema_name, revision = revision_date, + path = path }) + print(response.config) main.exit(0) end diff --git a/src/program/config/load/load.lua b/src/program/config/load/load.lua index bb72913efc..b69d20e79c 100644 --- a/src/program/config/load/load.lua +++ b/src/program/config/load/load.lua @@ -1,22 +1,17 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -local rpc = require("lib.yang.rpc") local common = require("program.config.common") function run(args) local schema_name, revision_date, instance_id, config = common.parse_command_line(args, { command='load', with_config_file=true }) - local caller = rpc.prepare_caller('snabb-config-leader-v1') local config_str = common.serialize_config(config, schema_name) - local data = { schema = schema_name, revision = revision_date, - config = config_str } - local msg, parse = rpc.prepare_call(caller, 'load-config', data) - local socket = common.open_socket_or_die(instance_id) - common.send_message(socket, msg) - local reply = common.recv_message(socket) - socket:close() - -- The reply is effectively empty. + local response = common.call_leader( + instance_id, 'load-config', + { schema = schema_name, revision = revision_date, + config = config_str }) + -- The reply is empty. main.exit(0) end From c2437309aa9b7a77887030300773f353506ec2ce Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 12:37:07 +0100 Subject: [PATCH 252/631] More snabb config refactor --- src/program/config/common.lua | 15 +++++++++------ src/program/config/get/get.lua | 9 ++++----- src/program/config/load/load.lua | 12 +++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index c02765d51d..a2d49d780a 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -36,30 +36,33 @@ function parse_command_line(args, opts) if not schema_name then err("missing --schema arg") end if #args == 0 then err() end local instance_id = table.remove(args, 1) - local ret = { schema_name, revision_date, instance_id } + local ret = { schema_name=schema_name, revision_date=revision_date, + instance_id=instance_id } if opts.with_config_file then if #args == 0 then err("missing config file argument") end local file = table.remove(args, 1) local opts = {schema_name=schema_name, revision_date=revision_date} - table.insert(ret, yang.load_configuration(file, opts)) + ret.config_file = file + ret.config = yang.load_configuration(file, opts) end if opts.with_path then if #args == 0 then err("missing path argument") end local path = table.remove(args, 1) -- Waiting on our XPath parsing library :) if path ~= '/' then err("paths other than / currently unimplemented") end - table.insert(ret, path) + ret.path = path end if opts.with_value then local parser = data_parser(schema_name, path) if #args == 0 then - table.insert(ret, parser(io.stdin:read('*a'))) + ret.value_str = io.stdin:read('*a') else - table.insert(ret, parser(table.remove(args, 1))) + ret.value_str = table.remove(args, 1) end + ret.value = parser(ret.value_str) end if #args ~= 0 then err("too many arguments") end - return unpack(ret) + return ret end function open_socket_or_die(instance_id) diff --git a/src/program/config/get/get.lua b/src/program/config/get/get.lua index b07dd13c25..9bbe6571ed 100644 --- a/src/program/config/get/get.lua +++ b/src/program/config/get/get.lua @@ -4,12 +4,11 @@ module(..., package.seeall) local common = require("program.config.common") function run(args) - local schema_name, revision_date, instance_id, path = - common.parse_command_line(args, { command='load', with_path=true }) + args = common.parse_command_line(args, { command='load', with_path=true }) local response = common.call_leader( - instance_id, 'get-config', - { schema = schema_name, revision = revision_date, - path = path }) + args.instance_id, 'get-config', + { schema = args.schema_name, revision = args.revision_date, + path = args.path }) print(response.config) main.exit(0) end diff --git a/src/program/config/load/load.lua b/src/program/config/load/load.lua index b69d20e79c..843ab6d05b 100644 --- a/src/program/config/load/load.lua +++ b/src/program/config/load/load.lua @@ -4,14 +4,12 @@ module(..., package.seeall) local common = require("program.config.common") function run(args) - local schema_name, revision_date, instance_id, config = - common.parse_command_line(args, - { command='load', with_config_file=true }) - local config_str = common.serialize_config(config, schema_name) + args = common.parse_command_line(args, + { command='load', with_config_file=true }) local response = common.call_leader( - instance_id, 'load-config', - { schema = schema_name, revision = revision_date, - config = config_str }) + args.instance_id, 'load-config', + { schema = args.schema_name, revision = args.revision_date, + config = common.serialize_config(args.config, args.schema_name) }) -- The reply is empty. main.exit(0) end From 5f8ba0fbda03520f16cd6930813ebc884294ea88 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 12:51:35 +0100 Subject: [PATCH 253/631] "snabb config" asks schema from leader if needed --- src/apps/config/leader.lua | 4 ++++ src/lib/yang/snabb-config-leader-v1.yang | 7 +++++++ src/program/config/common.lua | 24 ++++++++++++++---------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 9c56c40429..bce4a6d373 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -89,6 +89,10 @@ function Leader:enqueue_config_actions (actions) end end +function Leader:rpc_describe (args) + return { native_schema = self.schema_name } +end + function Leader:rpc_get_config (args) -- FIXME: Push more of this to a lib. assert(args.schema == self.schema_name) diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index 1aea50ce65..a9e00f338a 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -12,6 +12,13 @@ module snabb-config-leader-v1 { "Initial revision."; } + rpc describe { + output { + leaf native-schema { type string; mandatory true; } + // leaf-list alternate-schema { type string; } + } + } + rpc get-config { input { leaf schema { type string; mandatory true; } diff --git a/src/program/config/common.lua b/src/program/config/common.lua index a2d49d780a..859d59ad32 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -19,29 +19,33 @@ local parse_command_line_opts = { command = { required=true }, with_config_file = { default=false }, with_path = { default=false }, - with_value = { default=false } + with_value = { default=false }, + require_schema = { default=false } } function parse_command_line(args, opts) opts = lib.parse(opts, parse_command_line_opts) local function err(msg) show_usage(opts.command, 1, msg) end - local schema_name, revision_date + local ret = {} local handlers = {} function handlers.h() show_usage(opts.command, 0) end - function handlers.s(arg) schema_name = arg end - function handlers.r(arg) revision_date = arg end + function handlers.s(arg) ret.schema_name = arg end + function handlers.r(arg) ret.revision_date = arg end args = lib.dogetopt(args, handlers, "hs:r:", {help="h", ['schema-name']="s", schema="s", ['revision-date']="r", revision="r"}) - if not schema_name then err("missing --schema arg") end if #args == 0 then err() end - local instance_id = table.remove(args, 1) - local ret = { schema_name=schema_name, revision_date=revision_date, - instance_id=instance_id } + ret.instance_id = table.remove(args, 1) + if not ret.schema_name then + if opts.require_schema then err("missing --schema arg") end + ret.schema_name = + call_leader(ret.instance_id, 'describe', {}).native_schema + end if opts.with_config_file then if #args == 0 then err("missing config file argument") end local file = table.remove(args, 1) - local opts = {schema_name=schema_name, revision_date=revision_date} + local opts = {schema_name=ret.schema_name, + revision_date=ret.revision_date} ret.config_file = file ret.config = yang.load_configuration(file, opts) end @@ -53,7 +57,7 @@ function parse_command_line(args, opts) ret.path = path end if opts.with_value then - local parser = data_parser(schema_name, path) + local parser = data_parser(ret.schema_name, path) if #args == 0 then ret.value_str = io.stdin:read('*a') else From 830209d331072e7c43e8e18a26b074efb0fd7d2c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 12:57:12 +0100 Subject: [PATCH 254/631] Update program/config/README.md The -s option is optional. --- src/program/config/README.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/program/config/README.md b/src/program/config/README.md index 2288c7ee9d..1f8b5b50ad 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -40,13 +40,13 @@ like Sysrepo interacts with a Snabb network function. Most `snabb config` commands are invoked in a uniform way: ``` -snabb config SUBCOMMAND -m MODULE ID PATH [VALUE] +snabb config SUBCOMMAND [-s SCHEMA-NAME] ID PATH [VALUE] ``` `snabb config` speaks a data model that is based on YANG, for minimum impedance mismatch between NETCONF agents and Snabb applications. The -`-m MODULE` option allows the caller to indicate the YANG module that -they want to use, and for the purposes of the lwAFTR might be `-m +`-s SCHEMA-NAME` option allows the caller to indicate the YANG schema +that they want to use, and for the purposes of the lwAFTR might be `-s ietf-softwire`. `ID` identifies the particular Snabb instance to talk to, and can be a @@ -79,8 +79,10 @@ module snabb-simple-router { } ``` -In this case then, we would pass `-m snabb-simple-router` to all of -our `snabb config` invocations that talk to this router. +In this case then, we would pass `-s snabb-simple-router` to all of +our `snabb config` invocations that talk to this router. Snabb data +planes also declare their "native schema", so if you leave off the +`-s` option, `snabb config` will ask the data plane what schema it uses. The configuration for a Snabb instance can be expressed in a text format that is derived from the schema. In this case it could look @@ -101,7 +103,7 @@ and the YANG schema quoting rules for strings apply. Note that containers like `route {}` only appear in the data syntax if they are marked as `presence true;` in the schema. -So indeed, `snabb config get -m snabb-simple-router ID /` +So indeed, `snabb config get ID /` might print out just the output given above. Users can limit their query to a particular subtree via passing a @@ -109,7 +111,7 @@ different `PATH`. For example, with the same configuration, we can query just the `active` value: ``` -$ snabb config get -m snabb-simple-router ID /active +$ snabb config get ID /active true ``` @@ -122,7 +124,7 @@ To query an element of a `list` item, use an XPath selector; for example, to get the entry for IPv4 `1.2.3.4`, do: ``` -$ snabb config get -m snabb-simple-router ID /routes/route[addr=1.2.3.4] +$ snabb config get ID /routes/route[addr=1.2.3.4] route { addr 1.2.3.4; port 1; @@ -132,28 +134,28 @@ route { Or to just get the port: ``` -$ snabb config get -m snabb-simple-router ID /routes/route[addr=1.2.3.4]/port +$ snabb config get ID /routes/route[addr=1.2.3.4]/port 1 ``` Likewise, to change the port for `1.2.3.4`, do: ``` -$ snabb config set -m snabb-simple-router ID /routes/route[addr=1.2.3.4]/port 7 +$ snabb config set ID /routes/route[addr=1.2.3.4]/port 7 ``` Values can be large, so it's also possible to take them from `stdin`. Do this by omitting the value: ``` -$ cat /tmp/my-configuration | snabb config set -m snabb-simple-router ID / +$ cat /tmp/my-configuration | snabb config set ID / ``` Resetting the whole configuration is such a common operation that it has a special command that takes a file name instead of a path: ``` -$ snabb config load -m snabb-simple-router ID /tmp/my-configuration +$ snabb config load ID /tmp/my-configuration ``` Using `snabb config load` has the advantage that any configuration @@ -164,13 +166,13 @@ configuration that corresponds to YANG schema `leaf` or `leaf-list` nodes: ``` -$ snabb config delete -m snabb-simple-router ID /routes/route[addr=1.2.3.4] +$ snabb config delete ID /routes/route[addr=1.2.3.4] ``` One can of course augment a configuration as well: ``` -$ snabb config add -m snabb-simple-router ID /routes +$ snabb config add ID /routes route { addr 4.5.6.7; port 11; @@ -212,7 +214,7 @@ the reason for the error. These properties will be defined in the future. ``` -$ snabb config listen -m snabb-simple-router ID +$ snabb config listen -s snabb-simple-router ID { "id": "0", "verb": "get", "path": "/routes/route[addr=1.2.3.4]/port" } { "id": "1", "verb": "get", "path": "/routes/route[addr=2.3.4.5]/port" } { "id": "0", "status": "ok", "value: "1" } From 8372c4b3e2a974084aa975512275f6edc3b9d18e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 13:12:36 +0100 Subject: [PATCH 255/631] Implement "snabb config set" In the degenerate case :) --- src/apps/config/leader.lua | 3 ++- src/lib/yang/snabb-config-leader-v1.yang | 3 ++- src/program/config/common.lua | 10 +++++++++- src/program/config/load/load.lua | 2 +- src/program/config/set/README.inc | 15 +++++++++++++++ src/program/config/set/set.lua | 17 +++++++++++++++++ 6 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 src/program/config/set/README.inc create mode 100644 src/program/config/set/set.lua diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index bce4a6d373..b0c9627a18 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -104,8 +104,9 @@ function Leader:rpc_get_config (args) return { config = config_str } end -function Leader:rpc_load_config (args) +function Leader:rpc_set_config (args) assert(args.schema == self.schema_name) + assert(args.path == "/") local config = yang.load_data_for_schema_by_name(args.schema, args.config) self:reset_configuration(config) return {} diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index a9e00f338a..ee979c6518 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -30,10 +30,11 @@ module snabb-config-leader-v1 { } } - rpc load-config { + rpc set-config { input { leaf schema { type string; mandatory true; } leaf revision { type string; } + leaf path { type string; default "/"; } leaf config { type string; mandatory true; } } } diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 859d59ad32..4cfbd0cf79 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -23,6 +23,14 @@ local parse_command_line_opts = { require_schema = { default=false } } +local function data_parser(schema_name, path) + -- Waiting on XPath library. + assert(path == "/") + return function (str) + return data.load_data_for_schema_by_name(schema_name, str) + end +end + function parse_command_line(args, opts) opts = lib.parse(opts, parse_command_line_opts) local function err(msg) show_usage(opts.command, 1, msg) end @@ -57,7 +65,7 @@ function parse_command_line(args, opts) ret.path = path end if opts.with_value then - local parser = data_parser(ret.schema_name, path) + local parser = data_parser(ret.schema_name, ret.path) if #args == 0 then ret.value_str = io.stdin:read('*a') else diff --git a/src/program/config/load/load.lua b/src/program/config/load/load.lua index 843ab6d05b..9ef68d4c0b 100644 --- a/src/program/config/load/load.lua +++ b/src/program/config/load/load.lua @@ -7,7 +7,7 @@ function run(args) args = common.parse_command_line(args, { command='load', with_config_file=true }) local response = common.call_leader( - args.instance_id, 'load-config', + args.instance_id, 'set-config', { schema = args.schema_name, revision = args.revision_date, config = common.serialize_config(args.config, args.schema_name) }) -- The reply is empty. diff --git a/src/program/config/set/README.inc b/src/program/config/set/README.inc new file mode 100644 index 0000000000..cca7a54beb --- /dev/null +++ b/src/program/config/set/README.inc @@ -0,0 +1,15 @@ +Usage: + snabb config set [OPTIONS] ID PATH [VALUE] + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/set/set.lua b/src/program/config/set/set.lua new file mode 100644 index 0000000000..7e5227a827 --- /dev/null +++ b/src/program/config/set/set.lua @@ -0,0 +1,17 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local common = require("program.config.common") + +function run(args) + args = common.parse_command_line( + args, { command='set', with_path=true, with_value=true }) + local response = common.call_leader( + args.instance_id, 'set-config', + { schema = args.schema_name, revision = args.revision_date, + path = args.path, + config = common.serialize_config( + args.value, args.schema_name, args.path) }) + -- The reply is empty. + main.exit(0) +end From 8d7eb802d115da015ab04e3f3b4b2653712e44e8 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 14:51:00 +0100 Subject: [PATCH 256/631] Remove compile-binding-table, control lwaftr commands These are replaced by "snabb config". --- .../lwaftr/compile_binding_table/README | 99 ------------------- .../lwaftr/compile_binding_table/README.inc | 1 - .../compile_binding_table.lua | 32 ------ src/program/lwaftr/control/README | 13 --- src/program/lwaftr/control/README.inc | 1 - src/program/lwaftr/control/control.lua | 41 -------- 6 files changed, 187 deletions(-) delete mode 100644 src/program/lwaftr/compile_binding_table/README delete mode 120000 src/program/lwaftr/compile_binding_table/README.inc delete mode 100644 src/program/lwaftr/compile_binding_table/compile_binding_table.lua delete mode 100644 src/program/lwaftr/control/README delete mode 120000 src/program/lwaftr/control/README.inc delete mode 100644 src/program/lwaftr/control/control.lua diff --git a/src/program/lwaftr/compile_binding_table/README b/src/program/lwaftr/compile_binding_table/README deleted file mode 100644 index a861a391ee..0000000000 --- a/src/program/lwaftr/compile_binding_table/README +++ /dev/null @@ -1,99 +0,0 @@ -Usage: compile-binding-table BINDING-TABLE.TXT [BINDING-TABLE.O] - - -h, --help - Print usage information. - -Validate and compile an binding table. - -A binding table is a collection of softwires (tunnels). One endpoint -of the softwire is in the AFTR and the other is in the B4. A -softwire provisions an IPv4 address (or a part of an IPv4 address) to -a customer behind a B4. The B4 arranges for all IPv4 traffic to be -encapsulated in IPv6 and sent to the AFTR; the AFTR does the reverse. -The binding table is how the AFTR knows which B4 is associated with -an incoming packet. - -There are three parts of a binding table: the PSID info map, the -border router (BR) address table, and the softwire map. Grammatically -they appear in the file in the following order,: - - psid_map { - ... - } - br_addresses { - ... - } - softwires { - ... - } - -The PSID info map defines the set of IPv4 addresses that are provisioned -by an lwAFTR. It also defines the way in which those addresses are -shared, by specifying the "psid_length" and "shift" parameters for each -address. See RFC 7597 for more details on the PSID scheme for how to -share IPv4 addresses. The `psid_map' clause is composed of a list of -entries, each of which specifying a set of IPv4 address and the PSID -parameters. In this and other clauses, newlines and other white space -are insignificant. For example: - - psid_map { - 1.2.3.4 { psid_length=10 } - 5.6.7.8 { psid_length=5, shift=11 } - ... - } - -An entry's "psid_length" and "shift" parameters must necessarily add up -to 16, so it is sufficient to specify just one of them. If neither are -specified, they default to 0 and 16, respectively. - -The addresses may be specified as ranges or lists of ranges as well: - - psid_map { - 1.2.3.4 { psid_length=10 } - 2.0.0.0, 3.0.0.0, 4.0.0.0-4.1.2.3 { psid_length=7 } - } - -The set of IPv4 address ranges specified in the PSID info map must be -disjoint. - -Next, the `br_addresses' clause lists the set of IPv6 addresses to -associate with the lwAFTR. These are the "border router" addresses. -For a usual deployment there will be one main address and possibly some -additional ones. For example: - - br_addresses { - 8:9:a:b:c:d:e:f, - 1E:1:1:1:1:1:1:af, - 1E:2:2:2:2:2:2:af - } - -Finally, the `softwires' clause defines the set of softwires to -provision. Each softwire associates an IPv4 address, a PSID, and a B4 -address. For example: - - softwires { - { ipv4=178.79.150.233, psid=80, b4=127:2:3:4:5:6:7:128 } - { ipv4=178.79.150.233, psid=2300, b4=127:11:12:13:14:15:16:128 } - ... - } - -By default, a softwire is associated with the first entry in -`br_addresses' (aftr=0). To associate the tunnel with a different border router, -specify it by index: - - softwires { - { ipv4=178.79.150.233, psid=80, b4=127:2:3:4:5:6:7:128, aftr=0 } - { ipv4=178.79.150.233, psid=2300, b4=127:11:12:13:14:15:16:128, aftr=42 } - ... - } - -This command will atomically overwrite the output file. If the output -file name BINDING-TABLE.O is not given as an argument, the output file -will be the name of the input file, minus a ".txt" suffix if present, -plus a ".o" suffix. - -Note that when a lwAFTR is started, it will automatically compile its -binding table if needed. However for large tables (millions of entries) -this can take a second or two, so it can still be useful to compile the -binding table ahead of time. If you do this, you can send a SIGHUP to -the AFTR to cause it to reload the table. diff --git a/src/program/lwaftr/compile_binding_table/README.inc b/src/program/lwaftr/compile_binding_table/README.inc deleted file mode 120000 index 100b93820a..0000000000 --- a/src/program/lwaftr/compile_binding_table/README.inc +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/program/lwaftr/compile_binding_table/compile_binding_table.lua b/src/program/lwaftr/compile_binding_table/compile_binding_table.lua deleted file mode 100644 index 46bd6efa14..0000000000 --- a/src/program/lwaftr/compile_binding_table/compile_binding_table.lua +++ /dev/null @@ -1,32 +0,0 @@ -module(..., package.seeall) - -local lib = require('core.lib') -local stream = require("apps.lwaftr.stream") -local binding_table = require("apps.lwaftr.binding_table") - -function show_usage(code) - print(require("program.lwaftr.compile_binding_table.README_inc")) - main.exit(code) -end - -function parse_args(args) - local handlers = {} - function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "h", { help="h" }) - if #args < 1 or #args > 2 then show_usage(1) end - return unpack(args) -end - -function run(args) - local in_file, out_file = parse_args(args) - if not out_file then out_file = in_file:gsub("%.txt$", "")..'.o' end - -- We use the stream module because it gives us the mtime. - local input_stream = stream.open_input_text_stream(in_file) - local success, bt_or_err = pcall(binding_table.load_source, input_stream) - if not success then - io.stderr:write(tostring(bt_or_err)..'\n') - main.exit(1) - end - bt_or_err:save(out_file, stream.mtime_sec, stream.mtime_nsec) - main.exit(0) -end diff --git a/src/program/lwaftr/control/README b/src/program/lwaftr/control/README deleted file mode 100644 index 7764b52776..0000000000 --- a/src/program/lwaftr/control/README +++ /dev/null @@ -1,13 +0,0 @@ -Usage: control PID COMMAND ARG... - - -h, --help - Print usage information. - -Send a command to a running lwAFTR. Available commands include: - - reload - Tell an lwAFTR to reload its binding table. - - dump-configuration - Tell an lwAFTR to write out its configuration and binding table - to files in /tmp. diff --git a/src/program/lwaftr/control/README.inc b/src/program/lwaftr/control/README.inc deleted file mode 120000 index 100b93820a..0000000000 --- a/src/program/lwaftr/control/README.inc +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/program/lwaftr/control/control.lua b/src/program/lwaftr/control/control.lua deleted file mode 100644 index 8c62c6ccdf..0000000000 --- a/src/program/lwaftr/control/control.lua +++ /dev/null @@ -1,41 +0,0 @@ -module(..., package.seeall) - -local lib = require("core.lib") -local channel = require("apps.lwaftr.channel") -local messages = require("apps.lwaftr.messages") - -local command_names = { - reload = messages.lwaftr_message_reload, - dump_config = messages.lwaftr_message_reload -} - -function show_usage(code) - print(require("program.lwaftr.control.README_inc")) - main.exit(code) -end - -function parse_args(args) - local handlers = {} - function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "h", { help="h" }) - if #args ~= 2 then show_usage(1) end - local pid, command = unpack(args) - pid = tonumber(pid) - if not pid then show_usage(1) end - if command == 'reload' then - return pid, { messages.lwaftr_message_reload } - elseif command == 'dump-configuration' then - return pid, { messages.lwaftr_message_dump_config } - end - print('Unknown command: '..command) - show_usage(1) -end - -function run(args) - local pid, message = parse_args(args) - local ch = channel.open(pid, 'lwaftr/control', messages.lwaftr_message_t) - if ch:put(message) then main.exit(0) end - print(string.format( - 'Channel lwaftr/control for PID %d is full; try again later.', pid)) - main.exit(1) -end From c4be666d2e2736a456c9bbfa5d5f7d087ab469ce Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 15:10:46 +0100 Subject: [PATCH 257/631] Update generate-binding-table --- .../lwaftr/generate_binding_table/README | 28 ++++++++-- .../generate_binding_table.lua | 53 ++++++++----------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/src/program/lwaftr/generate_binding_table/README b/src/program/lwaftr/generate_binding_table/README index c3e1e9c7fd..ce42b8e9a9 100644 --- a/src/program/lwaftr/generate_binding_table/README +++ b/src/program/lwaftr/generate_binding_table/README @@ -1,14 +1,32 @@ Usage: -generate_binding_table - --output : Write down binding-table to file. +snabb lwaftr generate-binding-table + --output : Write down binding table to file. Examples: -- 10 public IPv4 address starting from 193.5.1.00. Total subscribers: 630 (10 * (2^6 - 1)): +- 10 public IPv4 address starting from 193.5.1.00. + Total subscribers: 630 (10 * (2^6 - 1)): -generate_binding_table --output binding-table.txt 193.5.1.100 10 fc00::100 fc00:1:2:3:4:5:0:7e 6 + snabb lwaftr generate-binding-table --output binding-table.txt \ + 193.5.1.100 10 fc00::100 fc00:1:2:3:4:5:0:7e 6 - 1 million subscribers: -generate_binding_table --output binding_table.txt 193.5.1.100 15874 fc00::100 fc00:1:2:3:4:5:0:7e 6 + snabb lwaftr generate-binding-table --output binding_table.txt \ + 193.5.1.100 15874 fc00::100 fc00:1:2:3:4:5:0:7e 6 + +The resulting binding table can be concatenated with +"external-interface" and "internal-interface" configuration blocks to +make a lwAFTR configuration. Here are some minimal configurations: + + external-interface { + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { mac 68:68:68:68:68:68; } + } + internal-interface { + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { mac 44:44:44:44:44:44; } + } diff --git a/src/program/lwaftr/generate_binding_table/generate_binding_table.lua b/src/program/lwaftr/generate_binding_table/generate_binding_table.lua index f65ab9b5a8..f93b2dc2b3 100644 --- a/src/program/lwaftr/generate_binding_table/generate_binding_table.lua +++ b/src/program/lwaftr/generate_binding_table/generate_binding_table.lua @@ -23,42 +23,28 @@ end local function psid_map_entry(v4addr, psid_len, shift) if tonumber(v4addr) then v4addr = to_ipv4_string(v4addr) end - return ("%s { psid_length=%d, shift=%d }"):format(v4addr, psid_len, shift) + return (" psid-map { addr %s; psid-length %d; shift %d; }"):format( + v4addr, psid_len, shift) end local function inc_ipv4(uint32) return uint32 + 1 end -local function psid_map_entries(params) - local entries = {} +local function psid_map(w, params) local v4addr = params.from_ipv4 if type(v4addr) == "string" then v4addr = to_ipv4_u32(v4addr) end assert(type(v4addr) == "number") for _ = 1, params.num_ips do - table.insert(entries, psid_map_entry(v4addr, params.psid_len, params.shift)) + w:ln(psid_map_entry(v4addr, params.psid_len, params.shift)) v4addr = inc_ipv4(v4addr) end - return entries -end - -local function psid_map(w, params) - w:ln("psid_map {") - for _, entry in ipairs(psid_map_entries(params)) do - w:ln(" "..entry) - end - w:ln("}") -end - -local function br_addresses(w, br_address) - w:ln("br_addresses {") - w:ln(" "..br_address) - w:ln("}") end local function softwire_entry(v4addr, psid_len, b4) if tonumber(v4addr) then v4addr = to_ipv4_string(v4addr) end - return ("{ ipv4=%s, psid=%d, b4=%s }"):format(v4addr, psid_len, b4) + return (" softwire { ipv4 %s; psid %d; b4-ipv6 %s; }"):format( + v4addr, psid_len, b4) end local function inc_ipv6(ipv6) @@ -75,10 +61,10 @@ end local function softwire_entries(from_ipv4, num_ips, psid_len, from_b4) local entries = {} - local v4addr = to_ipv4_u32(from_ipv4) - local b4 = ipv6:pton(from_b4) - local n = 2^psid_len - for _ = 1, num_ips do + local v4addr = to_ipv4_u32(params.from_ipv4) + local b4 = ipv6:pton(params.from_b4) + local n = 2^params.psid_len + for _ = 1, params.num_ips do for psid = 1, n-1 do table.insert(entries, softwire_entry(v4addr, psid, ipv6:ntop(b4))) b4 = inc_ipv6(b4) @@ -89,13 +75,16 @@ local function softwire_entries(from_ipv4, num_ips, psid_len, from_b4) end local function softwires(w, params) - w:ln("softwires {") - local entries = softwire_entries(params.from_ipv4, params.num_ips, - params.psid_len, params.from_b4) - for _, entry in ipairs(entries) do - w:ln(" "..entry) + local v4addr = to_ipv4_u32(params.from_ipv4) + local b4 = ipv6:pton(params.from_b4) + local n = 2^params.psid_len + for _ = 1, params.num_ips do + for psid = 1, n-1 do + w:ln(softwire_entry(v4addr, psid, ipv6:ntop(b4))) + b4 = inc_ipv6(b4) + end + v4addr = inc_ipv4(v4addr) end - w:ln("}") end local w = {} @@ -135,19 +124,21 @@ function run(args) if not shift then shift = 16 - psid_len end assert(psid_len + shift == 16) + w:ln("binding-table {") psid_map(w, { from_ipv4 = from_ipv4, num_ips = num_ips, psid_len = psid_len, shift = shift, }) - br_addresses(w, br_address) + w:ln(" br-address "..br_address..";") softwires(w, { from_ipv4 = from_ipv4, num_ips = num_ips, from_b4 = from_b4, psid_len = psid_len, }) + w:ln("}") w:close() main.exit(0) From 5316c14546163c9b552ed52e62e5cf1b5fe03739 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 17 Nov 2016 15:35:36 +0100 Subject: [PATCH 258/631] lwaftr supports run-time reconfiguration This adds the same thing to "lwaftr run" that we added to "lwaftr bench". --- src/program/lwaftr/bench/bench.lua | 4 +- src/program/lwaftr/run/README | 1 + src/program/lwaftr/run/run.lua | 68 ++++++++++++++++++++---------- src/program/lwaftr/setup.lua | 2 +- 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 8a2d99ba84..fa55c63db9 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -32,8 +32,8 @@ function run(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local graph = config.new() - setup.with_leader(setup.load_bench, graph, conf, - inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + setup.reconfigurable(setup.load_bench, graph, conf, + inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') app.configure(graph) local function start_sampling() diff --git a/src/program/lwaftr/run/README b/src/program/lwaftr/run/README index 8d0bfd7984..7475c1e5a2 100644 --- a/src/program/lwaftr/run/README +++ b/src/program/lwaftr/run/README @@ -9,6 +9,7 @@ Required arguments: --on-a-stick One single NIC for INET-side and B4-side Optional arguments: + --reconfigurable Allow run-time configuration query and update. --virtio Use virtio-net interfaces instead of Intel 82599 --ring-buffer-size Set Intel 82599 receive buffer size --cpu Bind the lwAFTR to the given CPU diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 9294a8c5e9..5ac78cb5fe 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -93,12 +93,14 @@ function parse_args(args) .." (valid values: flush, warn, off)") end end + function handlers.reconfigurable() opts.reconfigurable = true end function handlers.h() show_usage(0) end lib.dogetopt(args, handlers, "b:c:vD:yhir:", { conf = "c", v4 = 1, v6 = 1, ["v4-pci"] = 1, ["v6-pci"] = 1, verbose = "v", duration = "D", help = "h", virtio = "i", cpu = 1, ["ring-buffer-size"] = "r", ["real-time"] = 0, ["bench-file"] = "b", - ["ingress-drop-monitor"] = 1, ["on-a-stick"] = 1, mirror = 1, hydra = "y" }) + ["ingress-drop-monitor"] = 1, ["on-a-stick"] = 1, mirror = 1, + hydra = "y", reconfigurable = 0 }) if ring_buffer_size ~= nil then if opts.virtio_net then fatal("setting --ring-buffer-size does not work with --virtio") @@ -135,16 +137,22 @@ function run(args) local use_splitter = requires_splitter(opts, conf) local c = config.new() + local setup_fn, setup_args if opts.virtio_net then - setup.load_virt(c, conf, 'inetNic', v4, 'b4sideNic', v6) + setup_fn, setup_args = setup.load_virt, { 'inetNic', v4, 'b4sideNic', v6 } elseif opts["on-a-stick"] then - setup.load_on_a_stick(c, conf, { v4_nic_name = 'inetNic', - v6_nic_name = 'b4sideNic', - v4v6 = use_splitter and 'v4v6', - pciaddr = v4, - mirror = opts.mirror}) + setup_fn = setup.load_on_a_stick + setup_args = + { { v4_nic_name = 'inetNic', v6_nic_name = 'b4sideNic', + v4v6 = use_splitter and 'v4v6', pciaddr = v4, + mirror = opts.mirror } } else - setup.load_phy(c, conf, 'inetNic', v4, 'b4sideNic', v6) + setup_fn, setup_args = setup.load_phy, { 'inetNic', v4, 'b4sideNic', v6 } + end + if opts.reconfigurable then + setup.reconfigurable(setup_fn, c, conf, unpack(setup_args)) + else + setup_fn(c, conf, unpack(setup_args)) end engine.configure(c) @@ -154,26 +162,40 @@ function run(args) timer.activate(t) end + -- In reconfigurable mode, the app graph only gets populated later, + -- so we have to defer our timer creation. + local function later(f, when) + timer.activate(timer.new("later", f, when or 30e6)) + end + if opts.verbosity >= 1 then - local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) - -- Why are the names cross-referenced like this? - local ipv4_tx = opts.hydra and 'ipv4rx' or 'IPv4 RX' - local ipv4_rx = opts.hydra and 'ipv4tx' or 'IPv4 TX' - local ipv6_tx = opts.hydra and 'ipv6rx' or 'IPv6 RX' - local ipv6_rx = opts.hydra and 'ipv6tx' or 'IPv6 TX' - if use_splitter then - csv:add_app('v4v6', { 'v4', 'v4' }, { tx=ipv4_tx, rx=ipv4_rx }) - csv:add_app('v4v6', { 'v6', 'v6' }, { tx=ipv6_tx, rx=ipv6_rx }) - else - csv:add_app('inetNic', { 'tx', 'rx' }, { tx=ipv4_tx, rx=ipv4_rx }) - csv:add_app('b4sideNic', { 'tx', 'rx' }, { tx=ipv6_tx, rx=ipv6_rx }) + function add_csv_stats() + local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) + -- Link names like "tx" are from the app's perspective, but + -- these labels are from the perspective of the lwAFTR as a + -- whole so they are reversed. + local ipv4_tx = opts.hydra and 'ipv4rx' or 'IPv4 RX' + local ipv4_rx = opts.hydra and 'ipv4tx' or 'IPv4 TX' + local ipv6_tx = opts.hydra and 'ipv6rx' or 'IPv6 RX' + local ipv6_rx = opts.hydra and 'ipv6tx' or 'IPv6 TX' + if use_splitter then + csv:add_app('v4v6', { 'v4', 'v4' }, { tx=ipv4_tx, rx=ipv4_rx }) + csv:add_app('v4v6', { 'v6', 'v6' }, { tx=ipv6_tx, rx=ipv6_rx }) + else + csv:add_app('inetNic', { 'tx', 'rx' }, { tx=ipv4_tx, rx=ipv4_rx }) + csv:add_app('b4sideNic', { 'tx', 'rx' }, { tx=ipv6_tx, rx=ipv6_rx }) + end + csv:activate() end - csv:activate() + later(add_csv_stats) end if opts.ingress_drop_monitor then - local mon = ingress_drop_monitor.new({action=opts.ingress_drop_monitor}) - timer.activate(mon:timer()) + function add_ingress_drop_monitor() + local mon = ingress_drop_monitor.new({action=opts.ingress_drop_monitor}) + timer.activate(mon:timer()) + end + later(add_ingress_drop_monitor) end engine.busywait = true diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 85cde93f08..af072850a7 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -444,7 +444,7 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) link_sink(c, unpack(sinks)) end -function with_leader(f, graph, conf, ...) +function reconfigurable(f, graph, conf, ...) local args = {...} local function setup_fn(conf) local graph = config.new() From 20c3bb5073feb7179c9dd32f49f2a9b48cec04a8 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 18 Nov 2016 09:52:48 +0100 Subject: [PATCH 259/631] changelog updates --- src/program/lwaftr/doc/CHANGELOG.md | 30 ++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index e9e93bdb15..5788ae3cb4 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,26 +1,42 @@ # Change Log -## [3.0] - 2016-11-09 +## [3.0] - 2016-11-18 A change to migrate the lwAFTR to use a new YANG-based configuration. * New configuration format based on YANG. To migrate old configurations, run "snabb lwaftr migrate-configation old.conf" on - the old configuration. See the snabb-softwire-v1.yang schema or - README.configuration.md for full details on the new configuration - format. + the old configuration. See the [snabb-softwire-v1.yang + schema](../../../lib/yang/snabb-softwire-v1.yang) or + [README.configuration.md](./README.configuration.md) for full details + on the new configuration format. * Send ICMPv6 unreachable messages from the most appropriate source address available (the one associated with a B4 if possible, or else the one the packet one is in reply to had as a destination.) - * Add support for ARP resolution of the next hop on the external + * Add support for ARP resolution of the next hop on the external (IPv4) interface. - * Add support for virtualized control planes via Snabb vMX. + * Add support for virtualized control planes via Snabb vMX. See [the + `snabbvmx` documentation](../../snabbvmx/doc/README.md) for more. * Add many more counters, used to diagnose the path that packets take - in the lwAFTR. See README.counters.md for more. + in the lwAFTR. See [README.counters.md](./README.counters.md) for + more. + + * Add "snabb config" set of commands, to replace "snabb lwaftr control". + See (the `snabb config` documentation)[../../config/README.md) for + full details. + + * Add initial support for being able to reconfigure an entire lwAFTR + process while it is running, including changes that can add or remove + ingresss or egress filters, change NIC settings, or the like. Pass + the `--reconfigurable` argument to `snabb lwaftr run`, then interact + with the lwAFTR instance via `snabb config`. Enabling this option + currently has a small performance impact; this will go away in the + next release. A future release will also support efficient + incremental binding-table updates. * Many updates from upstream Snabb. From ff09ecb7b1186f6f8864967cfa1b279143ecf1ef Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 18 Nov 2016 09:56:00 +0100 Subject: [PATCH 260/631] Changelog markdown syntax fix --- src/program/lwaftr/doc/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 5788ae3cb4..bb73579813 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -26,7 +26,7 @@ A change to migrate the lwAFTR to use a new YANG-based configuration. more. * Add "snabb config" set of commands, to replace "snabb lwaftr control". - See (the `snabb config` documentation)[../../config/README.md) for + See [the `snabb config` documentation](../../config/README.md) for full details. * Add initial support for being able to reconfigure an entire lwAFTR From b148b4d899517d993129395198a661c7557dd336 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 18 Nov 2016 10:01:59 +0100 Subject: [PATCH 261/631] Syntax fix for README.configuration.md --- src/program/lwaftr/doc/README.configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/doc/README.configuration.md b/src/program/lwaftr/doc/README.configuration.md index 48f55fa088..808e82f0c5 100644 --- a/src/program/lwaftr/doc/README.configuration.md +++ b/src/program/lwaftr/doc/README.configuration.md @@ -143,7 +143,7 @@ address. See RFC 7597 for more details on the PSID scheme for how to share IPv4 addresses. IPv4 addresses are added to the `psid-map` with statements like this: -`` +``` psid-map { addr 178.79.150.3; end-addr 178.100.150.3; @@ -151,7 +151,7 @@ statements like this: reserved-ports-bit-count 0; shift 10; } -`` +``` `end-addr`, `reserved-ports-bit-count`, and `shift` are all optional. From 81c5abbb5247584b6ffbc0ce3c925c2f0344dff8 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Fri, 18 Nov 2016 10:59:58 +0100 Subject: [PATCH 262/631] Add Hydra mode to the lwaftr loadtest command (#583) Add Hydra mode to the lwaftr loadtest command --- src/program/lwaftr/bench/README | 2 +- src/program/lwaftr/csv_stats.lua | 18 +++--- src/program/lwaftr/loadtest/README | 11 ++++ src/program/lwaftr/loadtest/loadtest.lua | 70 +++++++++++++++++------- src/program/lwaftr/run/README | 2 +- src/program/packetblaster/lwaftr/README | 6 +- 6 files changed, 76 insertions(+), 33 deletions(-) diff --git a/src/program/lwaftr/bench/README b/src/program/lwaftr/bench/README index db7d4b208a..59f7d909b9 100644 --- a/src/program/lwaftr/bench/README +++ b/src/program/lwaftr/bench/README @@ -6,7 +6,7 @@ Usage: bench CONF IPV4-IN.PCAP IPV6-IN.PCAP Hydra mode: emit CSV data in the format expected by the Hydra reports. For instance: - benchmark,snabb,id,score,unit + benchmark,id,score,unit rather than the default: diff --git a/src/program/lwaftr/csv_stats.lua b/src/program/lwaftr/csv_stats.lua index 61950c42da..962e128918 100644 --- a/src/program/lwaftr/csv_stats.lua +++ b/src/program/lwaftr/csv_stats.lua @@ -17,15 +17,15 @@ CSVStatsTimer = {} -- -- Hydra mode example: -- --- benchmark,snabb,id,score,unit --- decap_mpps,master,1,3.362784,mpps --- decap_gbps,master,1,13.720160,gbps --- encap_mpps,master,1,3.362886,mpps --- encap_gbps,master,1,15.872824,gbps --- decap_mpps,master,2,3.407569,mpps --- decap_gbps,master,2,13.902880,gbps --- encap_mpps,master,2,3.407569,mpps --- encap_gbps,master,2,16.083724,gbps +-- benchmark,id,score,unit +-- decap_mpps,1,3.362784,mpps +-- decap_gbps,1,13.720160,gbps +-- encap_mpps,1,3.362886,mpps +-- encap_gbps,1,15.872824,gbps +-- decap_mpps,2,3.407569,mpps +-- decap_gbps,2,13.902880,gbps +-- encap_mpps,2,3.407569,mpps +-- encap_gbps,2,16.083724,gbps -- function CSVStatsTimer:new(filename, hydra_mode) local file = filename and io.open(filename, "w") or io.stdout diff --git a/src/program/lwaftr/loadtest/README b/src/program/lwaftr/loadtest/README index 5c731183b8..8c705697eb 100644 --- a/src/program/lwaftr/loadtest/README +++ b/src/program/lwaftr/loadtest/README @@ -13,6 +13,17 @@ Usage: loadtest [OPTIONS] [ 0, 'bitrate must be positive') assert(opts.step > 0, 'step must be positive') @@ -211,14 +214,13 @@ function run(args) end function tester.print_counter_diff( - before, after, duration, bench_file, gbps_bitrate) + before, after, duration, gbps_bitrate, bench_file, hydra_mode) local function bitrate(diff) -- 7 bytes preamble, 1 start-of-frame, 4 CRC, 12 interpacket gap. local overhead = 7 + 1 + 4 + 12 return (diff.txbytes + diff.txpackets * overhead) * 8 / duration end for _, stream in ipairs(streams) do - bench_file:write(('%f,%s'):format(gbps_bitrate, stream.tx_name)) print(string.format(' %s:', stream.tx_name)) local nic_id = stream.nic_tx_id local nic_before, nic_after = before[nic_id], after[nic_id] @@ -233,27 +235,55 @@ function run(args) local lost_percent = (tx.txpackets - rx.txpackets) / tx.txpackets * 100 print(string.format(' TX %d packets (%f MPPS), %d bytes (%f Gbps)', tx.txpackets, tx_mpps, tx.txbytes, tx_gbps)) - bench_file:write((',%d,%f,%d,%f'):format( - tx.txpackets, tx_mpps, tx.txbytes, tx_gbps)) print(string.format(' RX %d packets (%f MPPS), %d bytes (%f Gbps)', rx.txpackets, rx_mpps, rx.txbytes, rx_gbps)) - bench_file:write((',%d,%f,%d,%f'):format( - rx.txpackets, rx_mpps, rx.txbytes, rx_gbps)) print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', drop, lost_packets, lost_percent)) - bench_file:write((',%d,%d,%f\n'):format( - drop, lost_packets, lost_percent)) + if hydra_mode then + -- Hydra reports prefer integers for the X (time) axis. + -- TX + -- bench_file:write(('%s_tx_packets,%.f,%f,packets\n'):format( + -- stream.tx_name,gbps_bitrate,tx.txpackets)) + -- bench_file:write(('%s_tx_mpps,%.f,%f,mpps\n'):format( + -- stream.tx_name,gbps_bitrate,tx_mpps)) + -- bench_file:write(('%s_tx_bytes,%.f,%f,bytes\n'):format( + -- stream.tx_name,gbps_bitrate,tx.txbytes)) + -- bench_file:write(('%s_tx_gbps,%.f,%f,gbps\n'):format( + -- stream.tx_name,gbps_bitrate,tx_gbps)) + -- RX + -- bench_file:write(('%s_rx_packets,%.f,%f,packets\n'):format( + -- stream.tx_name,gbps_bitrate,rx.txpackets)) + bench_file:write(('%s_rx_mpps,%.f,%f,mpps\n'):format( + stream.tx_name,gbps_bitrate,rx_mpps)) + -- bench_file:write(('%s_rx_bytes,%.f,%f,bytes\n'):format( + -- stream.tx_name,gbps_bitrate,rx.txbytes)) + bench_file:write(('%s_rx_gbps,%.f,%f,gbps\n'):format( + stream.tx_name,gbps_bitrate,rx_gbps)) + -- Loss + bench_file:write(('%s_ingress_drop,%.f,%f,packets\n'):format( + stream.tx_name,gbps_bitrate,drop)) + -- bench_file:write(('%s_lost_packets,%.f,%f,packets\n'):format( + -- stream.tx_name,gbps_bitrate,lost_packets)) + bench_file:write(('%s_lost_percent,%.f,%f,percentage\n'):format( + stream.tx_name,gbps_bitrate,lost_percent)) + else + bench_file:write(('%f,%s,%d,%f,%d,%f,%d,%f,%d,%f,%d,%d,%f\n'):format( + gbps_bitrate, stream.tx_name, + tx.txpackets, tx_mpps, tx.txbytes, tx_gbps, + rx.txpackets, rx_mpps, rx.txbytes, rx_gbps, + drop, lost_packets, lost_percent)) + end end bench_file:flush() end - function tester.measure(bitrate, duration, bench_file) + function tester.measure(bitrate, duration, bench_file, hydra_mode) local gbps_bitrate = bitrate/1e9 local start_counters = tester.record_counters() local function report() local end_counters = tester.record_counters() - tester.print_counter_diff( - start_counters, end_counters, duration, bench_file, gbps_bitrate) + tester.print_counter_diff(start_counters, end_counters, duration, + gbps_bitrate, bench_file, hydra_mode) end print(string.format('Applying %f Gbps of load.', gbps_bitrate)) return tester.generate_load(bitrate, duration): @@ -262,10 +292,12 @@ function run(args) and_then(report) end - local function create_bench_file(filename) + local function create_bench_file(filename, hydra_mode) local bench_file = io.open(filename, "w") - bench_file:write("load_gbps,stream,tx_packets,tx_mpps,tx_bytes,tx_gbps".. - ",rx_packets,rx_mpps,rx_bytes,rx_gbps,ingress_drop,lost_packets,lost_percent\n") + local header = hydra_mode and "benchmark,id,score,unit\n" or + "load_gbps,stream,tx_packets,tx_mpps,tx_bytes,tx_gbps,rx_packets".. + ",rx_mpps,rx_bytes,rx_gbps,ingress_drop,lost_packets,lost_percent\n" + bench_file:write(header) bench_file:flush() return bench_file end @@ -280,7 +312,7 @@ function run(args) engine.main({done=done}) end - opts.bench_file = create_bench_file(opts.bench_file) + opts.bench_file = create_bench_file(opts.bench_file, opts.hydra) engine.busywait = true local head = promise.new() run_engine(head, diff --git a/src/program/lwaftr/run/README b/src/program/lwaftr/run/README index 8d0bfd7984..06331fb902 100644 --- a/src/program/lwaftr/run/README +++ b/src/program/lwaftr/run/README @@ -21,7 +21,7 @@ Optional arguments: Hydra mode: emit CSV data in the format expected by the Hydra reports. For instance: - benchmark,snabb,id,score,unit + benchmark,id,score,unit rather than the default: diff --git a/src/program/packetblaster/lwaftr/README b/src/program/packetblaster/lwaftr/README index 4c452a1db3..3ae59a2058 100644 --- a/src/program/packetblaster/lwaftr/README +++ b/src/program/packetblaster/lwaftr/README @@ -18,10 +18,10 @@ Usage: packetblaster lwaftr [OPTIONS] --src_mac SOURCE Source MAC-Address - Default: 00:00:00:00:00:00 + Default: 00:00:00:00:00:00 --dst_mac DESTINATION Destination MAC-Address - Default: 00:00:00:00:00:00 + Default: 00:00:00:00:00:00 --size SIZES A comma separated list of numbers. Send packets of SIZES bytes. The size specifies the lenght of the IPv4 @@ -83,7 +83,7 @@ Example 1: Measure performance of single stick LWAFTR (handling IPv4 and IPv6 traffic over a single interface). Packetblaster lwaftr generates 50% IPv4 and 50% IPv6 encapsulated traffic of IMIX line traffic: - $ sudo ./snabb packetblaster lwaftr --rate 3.2 --count 1000000000:05:00.0 + $ sudo ./snabb packetblaster lwaftr --rate 3.2 --count 100000 --pci 0000:05:00.0 packetblaster lwaftr: Sending 1000000 clients at 3.200 MPPS to 0000:05:00.0 IPv6: 2001:db8:: > 2001:db8:ffff::100: 10.0.0.0:1024 > 8.8.8.8:12345 From c0db10806f43b515428b4ad7e8597fd940b6dcdb Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 18 Nov 2016 11:45:45 +0100 Subject: [PATCH 263/631] Version 3.0.0 --- src/program/lwaftr/doc/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index bb73579813..f965c1ff34 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log -## [3.0] - 2016-11-18 +## [3.0.0] - 2016-11-18 A change to migrate the lwAFTR to use a new YANG-based configuration. From bfbc37f1dff844a68547f306569cf61312f0052e Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Fri, 18 Nov 2016 11:53:53 +0100 Subject: [PATCH 264/631] Add missing IMIX pcap files (#584) --- .../lwaftr/tests/benchdata/ipv4-imix.pcap | Bin 0 -> 3896 bytes .../lwaftr/tests/benchdata/ipv6-imix.pcap | Bin 0 -> 4256 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/program/lwaftr/tests/benchdata/ipv4-imix.pcap create mode 100644 src/program/lwaftr/tests/benchdata/ipv6-imix.pcap diff --git a/src/program/lwaftr/tests/benchdata/ipv4-imix.pcap b/src/program/lwaftr/tests/benchdata/ipv4-imix.pcap new file mode 100644 index 0000000000000000000000000000000000000000..beabe457bfe8ae0ac70df65ce117e0e4b4a46106 GIT binary patch literal 3896 zcmca|c+)~A1{MYw`2U}Qff2?5(tc1Z1O^ZZ4hB~S1_uzEUohC00|++xPh&K&1e&74 zz;N!@zx5Dxgh}#DM|K}RJIFJg37?Tja^#uLjAQ~nCV8eK;t`+O3=9cCPl7OdtTF|G z^zjP@S|i6QlNlsdS@9b>DmxkiI6{ED^u~t6v{B*F5WpV-kREp^nF0NT0& z*}wQ)Lt9rc;jK-+Z5hQroT;n5Jl9|E*Zm+bg0AC(;q0URM91T4}(7_%N{y+cer j&Uy!AFnT@CdH_<7bKtObRCqK5MnhmU1V%$(@P+^Y)nI-F literal 0 HcmV?d00001 From bf43c216f931ef5e957c2619842fb3a06350e693 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 18 Nov 2016 13:03:44 +0100 Subject: [PATCH 265/631] Fix nits and add resolve to lib.yang.xpath This fixes some spelling issues and some other small nits brought up in review. This also adds a resolve function which takes a schema, data tree and path and resolves the path for the data tree and corrisponding schema. --- src/lib/yang/xpath.lua | 169 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 17 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index 9563882847..a5f469066a 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -1,15 +1,15 @@ --- -- Use of this source code is governed by the Apache 2.0 license; see COPYING. +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. --- This module can be used to parse a path based on a yang schema (or it's +-- This module can be used to parse a path based on a yang schema (or its -- derivative grammar) and produce a lua table which is a native lua way -- of representing a path. The path provided is a subset of XPath supporting -- named keys such as [addr=1.2.3.4] and also basic positional querying -- for arrays e.g [position()=1] for the first element. -- --- The structure of the path is dependent on the type the node is, the +-- The structure of the path is dependent on the type the node is. The -- conversions are as follows: -- --- Scala fields: +-- Scalar fields: -- A lua string of the member name -- Struct fields: -- A lua string of the member name @@ -25,8 +25,13 @@ -- value as the value. module(..., package.seeall) -local schema = require("lib.yang.schema") -local data = require("lib.yang.data") +local ffi = require("ffi") +local lib = require("core.lib") +local schemalib = require("lib.yang.schema") +local datalib = require("lib.yang.data") +local valuelib = require("lib.yang.value") +local util = require("lib.yang.util") +local normalize_id = datalib.normalize_id local function extract_parts(fragment) local rtn = {query={}} @@ -47,9 +52,8 @@ end function handlers.table(fragment, tree) if #tree.keys == 1 then -- Check if the key is a string based key - k, v = pairs(tree.keys)() - if v.argument_type.primitive_type == "string" then - return {name=fragment.name, key=fragment.query[k]}, tree + if tree.string_key then + return {name=fragment.name, key=tree.string_key}, tree end else return {name=fragment.name, keys=fragment.query}, tree @@ -71,7 +75,7 @@ function next_element(path) end -- Converts an XPath path to a lua array consisting of path componants. --- A path compent can then be resolved on a yang data tree: +-- A path component can then be resolved on a yang data tree: local function convert_path(path, grammar) -- Extract head, check type and dispatch to handler. @@ -81,9 +85,9 @@ local function convert_path(path, grammar) local node if grammar.type == "table" then if grammar.keys[head] == nil then - node = assert(grammar.values[parts.name], err) + node = assert(grammar.values[parts.name], err) else - node = grammar.keys[head] + node = grammar.keys[head] end else node = assert(grammar[parts.name], err) @@ -98,13 +102,79 @@ local function convert_path(path, grammar) end end +function resolve(schema, data, path) + local handlers = {} + local function handle(scm, prod, data, path) + if #path == 0 then return data end + return assert(handlers[scm.kind], scm.kind)(scm, prod, data, path) + end + function handlers.list(scm, prod, data, path) + local head = table.remove(path, 1) + -- The list can either be a ctable or a plain old lua table, if it's the + -- ctable then it requires some more work to retrive the data. + local prod = prod.members[head.name] + local data = data[normalize_id(head.name)] + if #prod.keys > 1 then error("Multiple key values are not supported!") end + if prod.key_ctype then + -- It's a ctable and we need to prepare the key so we can lookup the + -- pointer to the entry and then convert the values back to lua. + local kparser = valuelib.types[scm.body[scm.key].primitive_type].parse + local key = kparser(head.keys[scm.key]) + local ckey = ffi.new(prod.key_ctype) + ckey[scm.key] = key + + data = data:lookup_ptr(ckey).value + else + data = data[normalize_id(head.keys[scm.key])] + end + + if #path == 0 then return data end + local peek = path[1] + if type(peek) == "table" then peek = peek.name end + scm = scm.body[peek] + prod = prod.values[peek] + return handle(scm, prod, data, path) + end + function handlers.container(scm, prod, data, path) + local head = table.remove(path, 1) + prod = prod.members[head] + data = data[normalize_id(head)] + if #path == 0 then return data end + local peek = path[1] + if type(peek) == "string" then scm = scm.body[peek] + else scm = scm.body[peek.name] end + return handle(scm, prod, data, path) + end + handlers["leaf-list"] = function (scm, prod, data, path) + local head = table.remove(path, 1) + if #path ~= 0 then error("Paths can't go beyond leaf-lists.") end + return data[normalize_id(head.name)][head.key] + end + function handlers.leaf(scm, prod, data, path) + local head = table.remove(path, 1) + if #path ~= 0 then error("Paths can't go beyond leaves.") end + return data[normalize_id(head)] + end + function handlers.module(scm, prod, data, path) + local peek = path[1] + if type(peek) == "table" then peek = peek.name end + scm = scm.body[peek] + return handle(scm, prod, data, path) + end + + local schema = lib.deepcopy(schema) + local path = lib.deepcopy(path) + local data = lib.deepcopy(data) + local grammar = datalib.data_grammar_from_schema(schema) + return handle(schema, grammar, data, path) +end -- Loads a module and converts the rest of the path. function load_from_path(path) -- First extract and load the module name then load it. - module_name, path = next_element(fragment) - scm = schema.load_schema_by_name(module_name) - grammar = data.data_grammar_for_schema(scm) + local module_name, path = next_element(path) + local scm = schema.load_schema_by_name(module_name) + local grammar = data.data_grammar_from_schema(scm) return module_name, convert_path(path, grammar.members) end @@ -128,8 +198,8 @@ function selftest() } }}]] - local scm = schema.load_schema(schema_src, "xpath-test") - local grammar = data.data_grammar_from_schema(scm) + local scm = schemalib.load_schema(schema_src, "xpath-test") + local grammar = datalib.data_grammar_from_schema(scm) -- Test path to lua path. local path = convert_path("/routes/route[addr=1.2.3.4]/port", grammar.members) @@ -143,4 +213,69 @@ function selftest() local path = convert_path("/blocked-ips[position()=4]/", grammar.members) assert(path[1].name == "blocked-ips") assert(path[1].key == 4) + + -- Test resolving a key to a path. + local data_src = [[ + active true; + + blocked-ips 8.8.8.8; + blocked-ips 8.8.4.4; + + routes { + route { addr 1.2.3.4; port 2; } + route { addr 2.3.4.5; port 2; } + route { addr 255.255.255.255; port 7; } + } + ]] + + local data = datalib.load_data_for_schema(scm, data_src) + + -- Try resolving a path in a list (ctable). + local path = convert_path("/routes/route[addr=1.2.3.4]/port", + grammar.members) + assert(resolve(scm, data, path) == 2) + + local path = convert_path("/routes/route[addr=255.255.255.255]/port", + grammar.members) + assert(resolve(scm, data, path) == 7) + + -- Try resolving a leaf-list + local path = convert_path("/blocked-ips[position()=1]", + grammar.members) + assert(resolve(scm, data, path) == util.ipv4_pton("8.8.8.8")) + + -- Try resolving a path for a list (non-ctable) + local fruit_schema_src = [[module fruit-bowl { + namespace snabb:fruit-bowl; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + container bowl { + presence true; + list fruit { + key name; + leaf name { type string; mandatory true; } + leaf rating { type uint8 { range 0..10; } mandatory true; } + } + }}]] + local fruit_data_src = [[ + bowl { + fruit { name "banana"; rating 10; } + fruit { name "pear"; rating 2; } + fruit { name "apple"; rating 6; } + } + ]] + + local fruit_scm = schemalib.load_schema(fruit_schema_src, "xpath-fruit-test") + local fruit_prod = datalib.data_grammar_from_schema(fruit_scm) + local fruit_data = datalib.load_data_for_schema(fruit_scm, fruit_data_src) + + local path = convert_path("/bowl/fruit[name=banana]/rating", + fruit_prod.members) + assert(resolve(fruit_scm, fruit_data, path) == 10) + + local path = convert_path("/bowl/fruit[name=apple]/rating", + fruit_prod.members) + assert(resolve(fruit_scm, fruit_data, path) == 6) end From 689e1f4deb90f2712ed7d89612f8adbbefc06dc6 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 18 Nov 2016 13:55:44 +0100 Subject: [PATCH 266/631] Fix invalid library references in lib.yang.xpath --- src/lib/yang/xpath.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index a5f469066a..cfcd9ad6b3 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -173,8 +173,8 @@ end function load_from_path(path) -- First extract and load the module name then load it. local module_name, path = next_element(path) - local scm = schema.load_schema_by_name(module_name) - local grammar = data.data_grammar_from_schema(scm) + local scm = schemalib.load_schema_by_name(module_name) + local grammar = datalib.data_grammar_from_schema(scm) return module_name, convert_path(path, grammar.members) end From 7164c27e2c80b5f3e83fbe93c5e1da1d469cd066 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 18 Nov 2016 17:39:12 +0100 Subject: [PATCH 267/631] Add on-a-stick mixed IPv4 IPv6 benchdata pcap Originally by tekniko, checksums fixed and filenames changed as per discussion --- .../benchdata/ipv4_and_ipv6_stick_imix.pcap | Bin 0 -> 8128 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/program/lwaftr/tests/benchdata/ipv4_and_ipv6_stick_imix.pcap diff --git a/src/program/lwaftr/tests/benchdata/ipv4_and_ipv6_stick_imix.pcap b/src/program/lwaftr/tests/benchdata/ipv4_and_ipv6_stick_imix.pcap new file mode 100644 index 0000000000000000000000000000000000000000..e65e8c537d2d80ead4410602ec44847ea709bc00 GIT binary patch literal 8128 zcmeHLJ5Iwu5Pdc_F<-|y0RKcev9dp0@wNcaDK!yuoVxRM3KFv&9 zTs=hP*myYx-oCz{+`PZ9w`Php>HY4b8k$#=q>fH7#1TsKt!q@#DKhL}7kk*p0S=K% z6c&L+lAs*_7lj{3sZt-)upG50{5Zya#Kc%wj#?Cc9Jd6IzQ?c}wJ7{Jw(Gl(p<9dL z)}t0hWq!CdnH6l#iDwplUfi6^oD-k9oRjub<0utaqrkn1HES~7+%6Q)G`mn6mvR?k zT&|Iuuaye;<-u*GcTS@MT?NACLB?fQZT7SZ_~pTY);;G@fsO)U^C080BQ`r~1wKuQ zYfaWEwk}SW) Date: Mon, 21 Nov 2016 10:47:29 +0100 Subject: [PATCH 268/631] Refactor lib.yang.xpath.convert_path and fix nits This completely refactors the convert_path function responsible for taking a yang xpath path and converting it to a lua path that can be used to resolve a node in a yang data tree. The refactor in general makes the code more readable by splitting some of the functionality out of the convert_path function into smaller, better defined functions. This also changes the return value from the resolve function to be a function which takes a data tree and resolves for that data tree. This allows the resolver to be re-used on multiple data trees with the same schema and path. --- src/lib/yang/xpath.lua | 99 +++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index cfcd9ad6b3..e86054cfdb 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -70,39 +70,52 @@ end -- Gets the next item in the path returning the element and the remaining -- path fragment. For example "router.routes.route" will return "router" -- and "routes.route". If the end is reached it'll return nil. -function next_element(path) +local function next_element(path) return string.match(path, "([^/]+)/?(.*)") end --- Converts an XPath path to a lua array consisting of path componants. --- A path component can then be resolved on a yang data tree: +local function split_path(path) + local tail = path + return function () + local head + head, tail = next_element(tail) + if head == nil then return head else return extract_parts(head) end + end +end -local function convert_path(path, grammar) - -- Extract head, check type and dispatch to handler. - local head, tail = next_element(path) - local parts = extract_parts(head) - local err = "Invalid path: "..parts.name - local node +-- Finds the grammar node for a fragment in a given grammar. +function extract_grammar_node(grammar, fragment) + local errmsg = "Invalid path: "..fragment.name if grammar.type == "table" then - if grammar.keys[head] == nil then - node = assert(grammar.values[parts.name], err) + if grammar.keys[fragment.name] == nil then + return assert(grammar.values[fragment.name], errmsg) else - node = grammar.keys[head] + return grammar.keys[fragment.name] end else - node = assert(grammar[parts.name], err) + return assert(grammar[fragment.name], errmsg) end - local element, node = handle(node.type, parts, node) - if tail ~= "" then - local rtn = convert_path(tail, node) - table.insert(rtn, 1, element) - return rtn - else - return {element} +end + +-- Converts an XPath path to a lua array consisting of path componants. +-- A path component can then be resolved on a yang data tree: +function convert_path(grammar, path) + local ret = {} + local node = grammar + for element in split_path(path) do + node = extract_grammar_node(node, element) + local luapath, next_node = handle(node.type, element, node) + table.insert(ret, luapath) + node = next_node end + return ret end -function resolve(schema, data, path) +-- Returns a resolver for a paticular schema and *lua* path. +function resolve(schema, path) + local schema = lib.deepcopy(schema) + local path = lib.deepcopy(path) + local grammar = datalib.data_grammar_from_schema(schema) local handlers = {} local function handle(scm, prod, data, path) if #path == 0 then return data end @@ -161,12 +174,10 @@ function resolve(schema, data, path) scm = scm.body[peek] return handle(scm, prod, data, path) end - - local schema = lib.deepcopy(schema) - local path = lib.deepcopy(path) - local data = lib.deepcopy(data) - local grammar = datalib.data_grammar_from_schema(schema) - return handle(schema, grammar, data, path) + return function (data) + local data = lib.deepcopy(data) + return handle(schema, grammar, data, path) + end end -- Loads a module and converts the rest of the path. @@ -175,7 +186,7 @@ function load_from_path(path) local module_name, path = next_element(path) local scm = schemalib.load_schema_by_name(module_name) local grammar = datalib.data_grammar_from_schema(scm) - return module_name, convert_path(path, grammar.members) + return module_name, convert_path(grammar.members, path) end function selftest() @@ -202,7 +213,7 @@ function selftest() local grammar = datalib.data_grammar_from_schema(scm) -- Test path to lua path. - local path = convert_path("/routes/route[addr=1.2.3.4]/port", grammar.members) + local path = convert_path(grammar.members,"/routes/route[addr=1.2.3.4]/port") assert(path[1] == "routes") assert(path[2].name == "route") @@ -210,7 +221,7 @@ function selftest() assert(path[2].keys["addr"] == "1.2.3.4") assert(path[3] == "port") - local path = convert_path("/blocked-ips[position()=4]/", grammar.members) + local path = convert_path(grammar.members, "/blocked-ips[position()=4]/") assert(path[1].name == "blocked-ips") assert(path[1].key == 4) @@ -231,18 +242,16 @@ function selftest() local data = datalib.load_data_for_schema(scm, data_src) -- Try resolving a path in a list (ctable). - local path = convert_path("/routes/route[addr=1.2.3.4]/port", - grammar.members) - assert(resolve(scm, data, path) == 2) + local path = convert_path(grammar.members,"/routes/route[addr=1.2.3.4]/port") + assert(resolve(scm, path)(data) == 2) - local path = convert_path("/routes/route[addr=255.255.255.255]/port", - grammar.members) - assert(resolve(scm, data, path) == 7) + local path = convert_path(grammar.members, + "/routes/route[addr=255.255.255.255]/port") + assert(resolve(scm, path)(data) == 7) -- Try resolving a leaf-list - local path = convert_path("/blocked-ips[position()=1]", - grammar.members) - assert(resolve(scm, data, path) == util.ipv4_pton("8.8.8.8")) + local path = convert_path(grammar.members,"/blocked-ips[position()=1]") + assert(resolve(scm, path)(data) == util.ipv4_pton("8.8.8.8")) -- Try resolving a path for a list (non-ctable) local fruit_schema_src = [[module fruit-bowl { @@ -271,11 +280,11 @@ function selftest() local fruit_prod = datalib.data_grammar_from_schema(fruit_scm) local fruit_data = datalib.load_data_for_schema(fruit_scm, fruit_data_src) - local path = convert_path("/bowl/fruit[name=banana]/rating", - fruit_prod.members) - assert(resolve(fruit_scm, fruit_data, path) == 10) + local path = convert_path(fruit_prod.members, + "/bowl/fruit[name=banana]/rating") + assert(resolve(fruit_scm, path)(fruit_data) == 10) - local path = convert_path("/bowl/fruit[name=apple]/rating", - fruit_prod.members) - assert(resolve(fruit_scm, fruit_data, path) == 6) + local path = convert_path(fruit_prod.members, + "/bowl/fruit[name=apple]/rating") + assert(resolve(fruit_scm, path)(fruit_data) == 6) end From fab5d4fa148894348833b00b25d33718ec8bd46d Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 21 Nov 2016 11:13:53 +0100 Subject: [PATCH 269/631] Fix final nits in lib.yang.xpath --- src/lib/yang/xpath.lua | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index e86054cfdb..22062fe377 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -50,14 +50,7 @@ function handlers.struct(fragment, tree) return fragment.name, tree.members end function handlers.table(fragment, tree) - if #tree.keys == 1 then - -- Check if the key is a string based key - if tree.string_key then - return {name=fragment.name, key=tree.string_key}, tree - end - else - return {name=fragment.name, keys=fragment.query}, tree - end + return {name=fragment.name, keys=fragment.query}, tree end function handlers.array(fragment, tree) local position = fragment.query["position()"] @@ -84,7 +77,7 @@ local function split_path(path) end -- Finds the grammar node for a fragment in a given grammar. -function extract_grammar_node(grammar, fragment) +local function extract_grammar_node(grammar, fragment) local errmsg = "Invalid path: "..fragment.name if grammar.type == "table" then if grammar.keys[fragment.name] == nil then From 45d8bb448876dffdd8d9a40cf7c54c455adf7664 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 21 Nov 2016 13:05:51 +0100 Subject: [PATCH 270/631] Fix some more nits in lib.yang.xpath brought up by review --- src/lib/yang/xpath.lua | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index 22062fe377..0fcc24b2fe 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -44,10 +44,10 @@ end local handlers = {} function handlers.scalar(fragment, tree) - return fragment.name, tree + return {name=fragment.name}, tree end function handlers.struct(fragment, tree) - return fragment.name, tree.members + return {name=fragment.name}, tree.members end function handlers.table(fragment, tree) return {name=fragment.name, keys=fragment.query}, tree @@ -78,6 +78,7 @@ end -- Finds the grammar node for a fragment in a given grammar. local function extract_grammar_node(grammar, fragment) + assert(grammar.type ~= "scalar") local errmsg = "Invalid path: "..fragment.name if grammar.type == "table" then if grammar.keys[fragment.name] == nil then @@ -98,6 +99,7 @@ function convert_path(grammar, path) for element in split_path(path) do node = extract_grammar_node(node, element) local luapath, next_node = handle(node.type, element, node) + luapath.grammar = node table.insert(ret, luapath) node = next_node end @@ -105,9 +107,7 @@ function convert_path(grammar, path) end -- Returns a resolver for a paticular schema and *lua* path. -function resolve(schema, path) - local schema = lib.deepcopy(schema) - local path = lib.deepcopy(path) +function resolver(schema, path) local grammar = datalib.data_grammar_from_schema(schema) local handlers = {} local function handle(scm, prod, data, path) @@ -136,19 +136,17 @@ function resolve(schema, path) if #path == 0 then return data end local peek = path[1] - if type(peek) == "table" then peek = peek.name end - scm = scm.body[peek] - prod = prod.values[peek] + scm = scm.body[peek.name] + prod = prod.values[peek.name] return handle(scm, prod, data, path) end function handlers.container(scm, prod, data, path) local head = table.remove(path, 1) - prod = prod.members[head] - data = data[normalize_id(head)] + prod = prod.members[head.name] + data = data[normalize_id(head.name)] if #path == 0 then return data end local peek = path[1] - if type(peek) == "string" then scm = scm.body[peek] - else scm = scm.body[peek.name] end + scm = scm.body[peek.name] return handle(scm, prod, data, path) end handlers["leaf-list"] = function (scm, prod, data, path) @@ -159,12 +157,11 @@ function resolve(schema, path) function handlers.leaf(scm, prod, data, path) local head = table.remove(path, 1) if #path ~= 0 then error("Paths can't go beyond leaves.") end - return data[normalize_id(head)] + return data[normalize_id(head.name)] end function handlers.module(scm, prod, data, path) local peek = path[1] - if type(peek) == "table" then peek = peek.name end - scm = scm.body[peek] + scm = scm.body[peek.name] return handle(scm, prod, data, path) end return function (data) @@ -208,11 +205,11 @@ function selftest() -- Test path to lua path. local path = convert_path(grammar.members,"/routes/route[addr=1.2.3.4]/port") - assert(path[1] == "routes") + assert(path[1].name == "routes") assert(path[2].name == "route") assert(path[2].keys) assert(path[2].keys["addr"] == "1.2.3.4") - assert(path[3] == "port") + assert(path[3].name == "port") local path = convert_path(grammar.members, "/blocked-ips[position()=4]/") assert(path[1].name == "blocked-ips") @@ -236,15 +233,15 @@ function selftest() -- Try resolving a path in a list (ctable). local path = convert_path(grammar.members,"/routes/route[addr=1.2.3.4]/port") - assert(resolve(scm, path)(data) == 2) + assert(resolver(scm, path)(data) == 2) local path = convert_path(grammar.members, "/routes/route[addr=255.255.255.255]/port") - assert(resolve(scm, path)(data) == 7) + assert(resolver(scm, path)(data) == 7) -- Try resolving a leaf-list local path = convert_path(grammar.members,"/blocked-ips[position()=1]") - assert(resolve(scm, path)(data) == util.ipv4_pton("8.8.8.8")) + assert(resolver(scm, path)(data) == util.ipv4_pton("8.8.8.8")) -- Try resolving a path for a list (non-ctable) local fruit_schema_src = [[module fruit-bowl { @@ -275,9 +272,9 @@ function selftest() local path = convert_path(fruit_prod.members, "/bowl/fruit[name=banana]/rating") - assert(resolve(fruit_scm, path)(fruit_data) == 10) + assert(resolver(fruit_scm, path)(fruit_data) == 10) local path = convert_path(fruit_prod.members, "/bowl/fruit[name=apple]/rating") - assert(resolve(fruit_scm, path)(fruit_data) == 6) + assert(resolver(fruit_scm, path)(fruit_data) == 6) end From 80e6ecaa639108f4ed438a0873b24e41bdfa11bb Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 21 Nov 2016 11:41:00 +0100 Subject: [PATCH 271/631] Allow ability to parse/print non-struct data Before, only structs were parseable and printable at the top level. However considering that we're going to be working with sub-data via the path API, we need to support parsing and printing sub-data. Structs print as before. Tables and arrays print as if they were in a struct with just that one keyword; that is, as a sequence of KEYWORD { ... } or KEYWORD VALUE; lines respectively. Sequences turn out to only be a top-level production so we can remove them from the sub-data grammar. The interesting thing comes in with scalars; we're now able to parse/print them without their keyword so that when you get a /foo/bar/baz and baz is a scalar, it prints just the value and not the "KEYWORD VALUE;" ceremony. --- src/lib/yang/data.lua | 124 +++++++++++++++++++++++++++------------- src/lib/yang/parser.lua | 9 +++ 2 files changed, 92 insertions(+), 41 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 3a48bb0b6e..61e338b6db 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -3,6 +3,7 @@ module(..., package.seeall) local parse_string = require("lib.yang.parser").parse_string +local decode_string = require("lib.yang.parser").decode_string local schema = require("lib.yang.schema") local util = require("lib.yang.util") local value = require("lib.yang.value") @@ -251,26 +252,6 @@ local function scalar_parser(keyword, argument_type, default, mandatory) return {init=init, parse=parse, finish=finish} end -local function sequence_parser(keyword, members) - local function init() return {} end - local function parse1(node) - local sub = assert(members[node.keyword], - 'unrecognized rpc: '..node.keyword) - return {id=node.keyword, data=sub.finish(sub.parse(node, sub.init()))} - end - local function parse(node, out) - assert(not node.keyword) -- ? - for _,node in ipairs(node.statements) do - table.insert(out, parse1(node)) - end - return out - end - local function finish(out) - return out - end - return {init=init, parse=parse, finish=finish} -end - local function ctable_builder(key_t, value_t) local res = ctable.new({ key_type=key_t, value_type=value_t }) local builder = {} @@ -379,15 +360,44 @@ function data_parser_from_grammar(production) return scalar_parser(keyword, production.argument_type, production.default, production.mandatory) end - function handlers.sequence(keyword, production) - return sequence_parser(keyword, visitn(production.members)) - end - local parser = visit1('(top level)', production) - return function(str, filename) - local node = {statements=parse_string(str, filename)} - return parser.finish(parser.parse(node, parser.init())) + local top_parsers = {} + function top_parsers.struct(production) + local parser = visit1('(top level)', production) + return function(str, filename) + local node = {statements=parse_string(str, filename)} + return parser.finish(parser.parse(node, parser.init())) + end end + function top_parsers.sequence(production) + local members = visitn(production.members) + return function(str, filename) + local ret = {} + for _, node in ipairs(parse_string(str, filename)) do + local sub = assert(members[node.keyword], + 'unrecognized rpc: '..node.keyword) + local data = sub.finish(sub.parse(node, sub.init())) + table.insert(ret, {id=node.keyword, data=data}) + end + return ret + end + end + local function generic_top_parser(production) + local parser = top_parsers.struct({ + type=struct, members={[production.keyword] = production}}) + return function(str, filename) + return parser(str, filename)[production.keyword] + end + end + top_parsers.array = generic_top_parser + top_parsers.table = generic_top_parser + function top_parsers.scalar(production) + local parse = value_parser(production.argument_type) + return function(str, filename) + return parse(decode_string(str, filename)) + end + end + return assert(top_parsers[production.type])(production) end function load_data_for_schema(schema, str, filename) @@ -551,35 +561,59 @@ local function data_printer_from_grammar(production) end end end - function handlers.sequence(keyword, production) + + local top_printers = {} + function top_printers.struct(production) + local printer = body_printer(production.members) + return function(data, file) + printer(data, file, '') + return file:flush() + end + end + function top_printers.sequence(production) local printers = {} for k,v in pairs(production.members) do printers[k] = printer(k, v) end - return function(data, file, indent) + return function(data, file) for _,elt in ipairs(data) do local id = assert(elt.id) - assert(printers[id])(elt.data, file, indent) + assert(printers[id])(elt.data, file, '') end + return file:flush() end end - - local top_printer = body_printer(production.members) - return function(data, file) - top_printer(data, file, '') - file:flush() + local function generic_top_printer(production) + local printer = body_printer({[production.keyword] = production}) + return function(data, file) + printer(data, file, '') + return file:flush() + end end + top_printers.array = generic_top_printer + top_printers.table = generic_top_printer + function top_printers.scalar(production) + local serialize = value_serializer(production.argument_type) + return function(data, file) + file:write(serialize(data)) + return file:flush() + end + end + return assert(top_printers[production.type])(production) +end + +local function string_output_file() + local file = {} + local out = {} + function file:write(str) table.insert(out, str) end + function file:flush(str) return table.concat(out) end + return file end function data_string_printer_from_grammar(production) local printer = data_printer_from_grammar(production) return function(data) - local file = {} - local out = {} - function file:write(str) table.insert(out, str) end - function file:flush(str) end - printer(data, file) - return table.concat(out) + return printer(data, string_output_file()) end end @@ -668,5 +702,13 @@ function selftest() file:close() os.remove(tmp) end + local scalar_uint32 = + { type='scalar', argument_type={primitive_type='uint32'} } + local parse_uint32 = data_parser_from_grammar(scalar_uint32) + local print_uint32 = data_printer_from_grammar(scalar_uint32) + assert(parse_uint32('1') == 1) + assert(parse_uint32('"1"') == 1) + assert(parse_uint32(' "1" \n ') == 1) + assert(print_uint32(1, string_output_file()) == '1') print('selfcheck: ok') end diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index e44b33f216..a29a2e753c 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -273,6 +273,15 @@ function Parser:parse_statement() self:error("Unexpected character found") end +function decode_string(str, filename) + local parser = Parser.new(str, filename) + parser:skip_whitespace() + local str = parser:parse_string() + parser:skip_whitespace() + if not parser:is_eof() then parser:error("Not end of file") end + return str +end + function parse_string(str, filename) local parser = Parser.new(str, filename) return parser:parse_module() From ad0c2d2fa43e266e54b370fd36ba2a3844ebc379 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 21 Nov 2016 12:37:57 +0100 Subject: [PATCH 272/631] Print to strings via string_output_file() --- src/apps/config/leader.lua | 5 +++-- src/lib/yang/data.lua | 21 ++++----------------- src/lib/yang/rpc.lua | 5 +++-- src/lib/yang/util.lua | 8 ++++++++ src/lib/yang/yang.lua | 2 ++ src/program/config/common.lua | 7 ++----- 6 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 97800a21a0..5bcf903143 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -99,8 +99,9 @@ function Leader:rpc_get_config (args) assert(args.path == '/') local schema = yang.load_schema_by_name(self.schema_name) local grammar = data.data_grammar_from_schema(schema) - local printer = data.data_string_printer_from_grammar(grammar) - local config_str = printer(self.current_configuration) + local printer = data.data_printer_from_grammar(grammar) + local config_str = printer(self.current_configuration, + yang.string_output_file()) return { config = config_str } end diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 61e338b6db..e6a9c3ed07 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -410,11 +410,11 @@ function load_data_for_schema_by_name(schema_name, str, filename) end function rpc_input_parser_from_schema(schema) - return data_parser_from_grammar(rpc_grammar_from_schema(schema).input) + return data_parser_from_grammar(rpc_input_grammar_from_schema(schema)) end function rpc_output_parser_from_schema(schema) - return data_parser_from_grammar(rpc_grammar_from_schema(schema).output) + return data_parser_from_grammar(rpc_output_grammar_from_schema(schema)) end local function encode_yang_string(str) @@ -610,13 +610,6 @@ local function string_output_file() return file end -function data_string_printer_from_grammar(production) - local printer = data_printer_from_grammar(production) - return function(data) - return printer(data, string_output_file()) - end -end - function data_printer_from_schema(schema) return data_printer_from_grammar(data_grammar_from_schema(schema)) end @@ -630,18 +623,12 @@ function print_data_for_schema_by_name(schema_name, data, file) return print_data_for_schema(schema, data, file) end -function rpc_printer_from_grammar(production) - local printer = data_string_printer_from_grammar( - { type='struct', members = { rpcs=production } }) - return function(rpcs) return printer({rpcs=rpcs}) end -end - function rpc_input_printer_from_schema(schema) - return rpc_printer_from_grammar(rpc_grammar_from_schema(schema).input) + return data_printer_from_grammar(rpc_input_grammar_from_schema(schema)) end function rpc_output_printer_from_schema(schema) - return rpc_printer_from_grammar(rpc_grammar_from_schema(schema).output) + return data_printer_from_grammar(rpc_output_grammar_from_schema(schema)) end function selftest() diff --git a/src/lib/yang/rpc.lua b/src/lib/yang/rpc.lua index 8556c3dd47..0d2fde5972 100644 --- a/src/lib/yang/rpc.lua +++ b/src/lib/yang/rpc.lua @@ -4,6 +4,7 @@ module(..., package.seeall) local schema = require("lib.yang.schema") local data = require("lib.yang.data") +local util = require("lib.yang.util") function prepare_callee(schema_name) local schema = schema.load_schema_by_name(schema_name) @@ -22,7 +23,7 @@ function prepare_caller(schema_name) end function prepare_calls(caller, calls) - local str = caller.print_input(calls) + local str = caller.print_input(calls, util.string_output_file()) local function parse_responses(str) local responses = caller.parse_output(str) assert(#responses == #calls) @@ -48,7 +49,7 @@ function handle_calls(callee, str, handle) table.insert(responses, { id=call.id, data=handle(call.id, call.data) }) end - return callee.print_output(responses) + return callee.print_output(responses, util.string_output_file()) end function dispatch_handler(obj, prefix) diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 7809f2fd55..339f5247d7 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -80,6 +80,14 @@ function ipv4_ntop(addr) return ipv4:ntop(ffi.new('uint32_t[1]', lib.htonl(addr))) end +function string_output_file() + local file = {} + local out = {} + function file:write(str) table.insert(out, str) end + function file:flush(str) return table.concat(out) end + return file +end + function selftest() print('selftest: lib.yang.util') assert(tointeger('0') == 0) diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 172a1669d4..2e3cc22bba 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -6,6 +6,7 @@ local schema = require("lib.yang.schema") local data = require("lib.yang.data") local binary = require("lib.yang.binary") local stream = require("lib.yang.stream") +local util = require("lib.yang.util") load_schema = schema.load_schema load_schema_file = schema.load_schema_file @@ -16,6 +17,7 @@ load_data_for_schema_by_name = data.load_data_for_schema_by_name print_data_for_schema = data.print_data_for_schema print_data_for_schema_by_name = data.print_data_for_schema_by_name +string_output_file = util.string_output_file compile_data_for_schema = binary.compile_data_for_schema compile_data_for_schema_by_name = binary.compile_data_for_schema_by_name diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 4cfbd0cf79..be7531a931 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -94,11 +94,8 @@ end function serialize_config(config, schema_name, path) assert(path == nil or path == "/") - -- FFS - local schema = yang.load_schema_by_name(schema_name) - local grammar = data.data_grammar_from_schema(schema) - local printer = data.data_string_printer_from_grammar(grammar) - return printer(config) + return yang.print_data_for_schema_by_name(config, schema_name, + yang.string_output_file()) end function send_message(socket, msg_str) From c0eceb0738e2e961789a68fa8238ce31c97f73e5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 21 Nov 2016 14:23:53 +0100 Subject: [PATCH 273/631] Prepare "snabb config get" for paths --- src/apps/config/leader.lua | 33 +++++++++++++++++++++++++-------- src/lib/yang/data.lua | 2 +- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 5bcf903143..1bbee65c45 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -93,16 +93,33 @@ function Leader:rpc_describe (args) return { native_schema = self.schema_name } end +local function path_getter_for_grammar(grammar, path) + -- Implement me :) + assert(path == '/') + return function(data) return data end, grammar +end + +local function path_printer_for_grammar(grammar, path) + local getter, subgrammar = path_getter_for_grammar(grammar, path) + local printer = data.data_printer_from_grammar(subgrammar) + return function(data, file) + return printer(getter(data), file) + end +end + +local function path_printer_for_schema(schema, path) + return path_printer_for_grammar(data.data_grammar_from_schema(schema), path) +end + +local function path_printer_for_schema_by_name(schema_name, path) + return path_printer_for_schema(yang.load_schema_by_name(schema_name), path) +end + function Leader:rpc_get_config (args) - -- FIXME: Push more of this to a lib. assert(args.schema == self.schema_name) - assert(args.path == '/') - local schema = yang.load_schema_by_name(self.schema_name) - local grammar = data.data_grammar_from_schema(schema) - local printer = data.data_printer_from_grammar(grammar) - local config_str = printer(self.current_configuration, - yang.string_output_file()) - return { config = config_str } + local printer = path_printer_for_schema_by_name(args.schema, args.path) + local config = printer(self.current_configuration, yang.string_output_file()) + return { config = config } end function Leader:rpc_set_config (args) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index e6a9c3ed07..95836c461a 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -450,7 +450,7 @@ local function value_serializer(typ) return serializer end -local function data_printer_from_grammar(production) +function data_printer_from_grammar(production) local handlers = {} local function printer(keyword, production) return assert(handlers[production.type])(keyword, production) From 3685310e9c8398cc5b15f6738e30583946e74841 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 21 Nov 2016 16:09:08 +0100 Subject: [PATCH 274/631] Fix serialization in program.config.common Broken in last commit. --- src/program/config/common.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index be7531a931..9ba1d15f35 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -94,7 +94,7 @@ end function serialize_config(config, schema_name, path) assert(path == nil or path == "/") - return yang.print_data_for_schema_by_name(config, schema_name, + return yang.print_data_for_schema_by_name(schema_name, config, yang.string_output_file()) end From 3a55f9f102b8a2142c54bd684bcde8804f13566f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 21 Nov 2016 16:09:51 +0100 Subject: [PATCH 275/631] Prepare "snabb config set" for paths Also add the concept of "schema support", a schema-specific module that can shortcut the actions to mutate an app graph. --- src/apps/config/leader.lua | 54 +++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 1bbee65c45..40c50057c2 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -122,11 +122,59 @@ function Leader:rpc_get_config (args) return { config = config } end +local generic_schema_support = { + compute_config_actions = function(old_graph, new_graph, verb, path, subconf) + return app.compute_config_actions(old_graph, new_graph) + end +} + +local function load_schema_support(schema_name) + return generic_schema_support +end + +local function path_parser_for_grammar(grammar, path) + local getter, subgrammar = path_getter_for_grammar(grammar, path) + return data.data_parser_from_grammar(subgrammar) +end + +local function path_parser_for_schema(schema, path) + return path_parser_for_grammar(data.data_grammar_from_schema(schema), path) +end + +local function path_parser_for_schema_by_name(schema_name, path) + return path_parser_for_schema(yang.load_schema_by_name(schema_name), path) +end + +local function path_setter_for_grammar(grammar, path) + -- Implement me :) + assert(path == "/") + return function(config, subconfig) + return subconfig + end +end + +local function path_setter_for_schema(schema, path) + return path_setter_for_grammar(data.data_grammar_from_schema(schema), path) +end + +local function path_setter_for_schema_by_name(schema_name, path) + return path_setter_for_schema(yang.load_schema_by_name(schema_name), path) +end + function Leader:rpc_set_config (args) assert(args.schema == self.schema_name) - assert(args.path == "/") - local config = yang.load_data_for_schema_by_name(args.schema, args.config) - self:reset_configuration(config) + local parser = path_parser_for_schema_by_name(args.schema, args.path) + local setter = path_setter_for_schema_by_name(args.schema, args.path) + local subconfig = parser(args.config) + local new_config = setter(self.configuration, subconfig) + local new_app_graph = self.setup_fn(new_config) + local support = load_schema_support(args.schema) + local actions = support.compute_config_actions(self.current_app_graph, + new_app_graph, 'set', + args.path, subconfig) + self:enqueue_config_actions(actions) + self.current_app_graph = new_app_graph + self.current_configuration = new_config return {} end From 791f1a0ce44e9fb3d93d5d47302c2eb7f72ff410 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 21 Nov 2016 16:30:01 +0100 Subject: [PATCH 276/631] Add stub "snabb config add", "snabb config remove" Also prepare program.config.common for paths. --- src/lib/yang/snabb-config-leader-v1.yang | 17 +++++++++++ src/program/config/README.md | 38 ++++++++++++------------ src/program/config/add/README.inc | 15 ++++++++++ src/program/config/add/add.lua | 17 +++++++++++ src/program/config/common.lua | 8 +++-- src/program/config/load/load.lua | 3 +- src/program/config/remove/README.inc | 15 ++++++++++ src/program/config/remove/remove.lua | 15 ++++++++++ src/program/config/set/set.lua | 2 +- 9 files changed, 106 insertions(+), 24 deletions(-) create mode 100644 src/program/config/add/README.inc create mode 100644 src/program/config/add/add.lua create mode 100644 src/program/config/remove/README.inc create mode 100644 src/program/config/remove/remove.lua diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index ee979c6518..340e076d32 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -38,4 +38,21 @@ module snabb-config-leader-v1 { leaf config { type string; mandatory true; } } } + + rpc add-config { + input { + leaf schema { type string; mandatory true; } + leaf revision { type string; } + leaf path { type string; mandatory true; } + leaf config { type string; mandatory true; } + } + } + + rpc remove-config { + input { + leaf schema { type string; mandatory true; } + leaf revision { type string; } + leaf path { type string; mandatory true; } + } + } } diff --git a/src/program/config/README.md b/src/program/config/README.md index 1f8b5b50ad..039cd9d7c8 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -12,28 +12,28 @@ the existing configuration of a Snabb instance. `snabb config` is a family of Snabb commands. Its sub-commands include: -* [`snabb config get`](./get/README.md): read configuration data +* [`snabb config get`](./get/README_inc): read configuration data -* [`snabb config get-state`](./get_state/README.md): read state data +* [`snabb config get-state`](./get_state/README_inc): read state data -* [`snabb config load`](./load/README.md): load a new configuration +* [`snabb config load`](./load/README_inc): load a new configuration -* [`snabb config set`](./set/README.md): incrementally update configuration +* [`snabb config set`](./set/README_inc): incrementally update configuration -* [`snabb config add`](./add/README.md): augment configuration, for +* [`snabb config add`](./add/README_inc): augment configuration, for example by adding a routing table entry -* [`snabb config delete`](./delete/README.md): remove a component from +* [`snabb config remove`](./delete/README_inc): remove a component from a configuration, for example removing a routing table entry -* [`snabb config listen`](./listen/README.md): provide an interface to - the `snabb config` functionality over a persistent socket, to - minimize per-operation cost +* [`snabb config listen`](./listen/README_inc): provide an interface to + the `snabb config` functionality over a persistent socket, to minimize + per-operation cost -The `snabb config get` commands are the normal way that Snabb users -interact with Snabb applications in an ad-hoc fashion via the command -line. `snabb config listen` is the standard way that a NETCONF agent -like Sysrepo interacts with a Snabb network function. +The `snabb config get` et al commands are the normal way that Snabb +users interact with Snabb applications in an ad-hoc fashion via the +command line. `snabb config listen` is the standard way that a NETCONF +agent like Sysrepo interacts with a Snabb network function. ### Configuration model @@ -161,12 +161,12 @@ $ snabb config load ID /tmp/my-configuration Using `snabb config load` has the advantage that any configuration error has a corresponding source location. -`snabb config` can also delete part of a configuration, but only on +`snabb config` can also remove part of a configuration, but only on configuration that corresponds to YANG schema `leaf` or `leaf-list` nodes: ``` -$ snabb config delete ID /routes/route[addr=1.2.3.4] +$ snabb config remove ID /routes/route[addr=1.2.3.4] ``` One can of course augment a configuration as well: @@ -189,7 +189,7 @@ following properties: listen`; just a convenience for the other side. - `verb`: The action to perform; one of `get-state`, `get`, `set`, - `add`, or `delete`. A string. + `add`, or `remove`. A string. - `path`: A path identifying the configuration or state data on which to operate. A string. @@ -255,6 +255,6 @@ updates to the data plane in an efficient way. See the [`apps.config` documentation](../../apps/config/README.md) for full details. Some data planes, like the lwAFTR, add hooks to the `set`, `add`, and -`delete` subcommands of `snabb config` to allow even more efficient -incremental updates, for example updating the binding table in place -via a custom protocol. +`remove` subcommands of `snabb config` to allow even more efficient +incremental updates, for example updating the binding table in place via +a custom protocol. diff --git a/src/program/config/add/README.inc b/src/program/config/add/README.inc new file mode 100644 index 0000000000..29eb9544f6 --- /dev/null +++ b/src/program/config/add/README.inc @@ -0,0 +1,15 @@ +Usage: + snabb config add [OPTIONS] ID PATH [VALUE] + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/add/add.lua b/src/program/config/add/add.lua new file mode 100644 index 0000000000..cd6d4bb0f4 --- /dev/null +++ b/src/program/config/add/add.lua @@ -0,0 +1,17 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local common = require("program.config.common") + +function run(args) + args = common.parse_command_line( + args, { command='add', with_path=true, with_value=true }) + local response = common.call_leader( + args.instance_id, 'add-config', + { schema = args.schema_name, revision = args.revision_date, + path = args.path, + config = common.serialize_config( + args.value, args.schema_name, args.path, 'add') }) + -- The reply is empty. + main.exit(0) +end diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 9ba1d15f35..c7a0ec2164 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -23,9 +23,10 @@ local parse_command_line_opts = { require_schema = { default=false } } -local function data_parser(schema_name, path) +local function data_parser(schema_name, path, command) -- Waiting on XPath library. assert(path == "/") + assert(command ~= 'add') return function (str) return data.load_data_for_schema_by_name(schema_name, str) end @@ -65,7 +66,7 @@ function parse_command_line(args, opts) ret.path = path end if opts.with_value then - local parser = data_parser(ret.schema_name, ret.path) + local parser = data_parser(ret.schema_name, ret.path, opts.command) if #args == 0 then ret.value_str = io.stdin:read('*a') else @@ -92,8 +93,9 @@ function open_socket_or_die(instance_id) return socket end -function serialize_config(config, schema_name, path) +function serialize_config(config, schema_name, path, command) assert(path == nil or path == "/") + assert(command ~= 'add') return yang.print_data_for_schema_by_name(schema_name, config, yang.string_output_file()) end diff --git a/src/program/config/load/load.lua b/src/program/config/load/load.lua index 9ef68d4c0b..38c4bad24d 100644 --- a/src/program/config/load/load.lua +++ b/src/program/config/load/load.lua @@ -9,7 +9,8 @@ function run(args) local response = common.call_leader( args.instance_id, 'set-config', { schema = args.schema_name, revision = args.revision_date, - config = common.serialize_config(args.config, args.schema_name) }) + config = common.serialize_config(args.config, args.schema_name, + 'load') }) -- The reply is empty. main.exit(0) end diff --git a/src/program/config/remove/README.inc b/src/program/config/remove/README.inc new file mode 100644 index 0000000000..212726b901 --- /dev/null +++ b/src/program/config/remove/README.inc @@ -0,0 +1,15 @@ +Usage: + snabb config remove [OPTIONS] ID PATH + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/remove/remove.lua b/src/program/config/remove/remove.lua new file mode 100644 index 0000000000..9f6ee297b3 --- /dev/null +++ b/src/program/config/remove/remove.lua @@ -0,0 +1,15 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local common = require("program.config.common") + +function run(args) + args = common.parse_command_line( + args, { command='remove', with_path=true }) + local response = common.call_leader( + args.instance_id, 'remove-config', + { schema = args.schema_name, revision = args.revision_date, + path = args.path }) + -- The reply is empty. + main.exit(0) +end diff --git a/src/program/config/set/set.lua b/src/program/config/set/set.lua index 7e5227a827..9ce0a583c6 100644 --- a/src/program/config/set/set.lua +++ b/src/program/config/set/set.lua @@ -11,7 +11,7 @@ function run(args) { schema = args.schema_name, revision = args.revision_date, path = args.path, config = common.serialize_config( - args.value, args.schema_name, args.path) }) + args.value, args.schema_name, args.path, 'set') }) -- The reply is empty. main.exit(0) end From ca390fe575ecbfddac15b033c758e439a5901644 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 21 Nov 2016 16:40:21 +0100 Subject: [PATCH 277/631] Tighten up definitions in lib.yang.xpath This refactors a bunch of code to tighten up the definitions of functions that are in lib.yang.xpath. These do more checking, are easier to read and more versatile. I've removed arguments which didn't *have* to be there. --- src/lib/yang/xpath.lua | 120 +++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 70 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index 0fcc24b2fe..a8e0d4443d 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -26,7 +26,6 @@ module(..., package.seeall) local ffi = require("ffi") -local lib = require("core.lib") local schemalib = require("lib.yang.schema") local datalib = require("lib.yang.data") local valuelib = require("lib.yang.value") @@ -43,21 +42,21 @@ local function extract_parts(fragment) end local handlers = {} -function handlers.scalar(fragment, tree) - return {name=fragment.name}, tree +function handlers.scalar(grammar, fragment) + return {name=fragment.name, grammar=grammar} end -function handlers.struct(fragment, tree) - return {name=fragment.name}, tree.members +function handlers.struct(grammar, fragment) + return {name=fragment.name, grammar=grammar} end -function handlers.table(fragment, tree) - return {name=fragment.name, keys=fragment.query}, tree +function handlers.table(grammar, fragment) + return {name=fragment.name, keys=fragment.query, grammar=grammar} end -function handlers.array(fragment, tree) +function handlers.array(grammar, fragment) local position = fragment.query["position()"] - return {name=fragment.name, key=tonumber(position)} + return {name=fragment.name, key=tonumber(position), grammar=grammar} end -function handle(node_type, fragment, tree) - return assert(handlers[node_type], node_type)(fragment, tree) +function handle(grammar, fragment) + return assert(handlers[grammar.type], grammar.type)(grammar, fragment) end -- Gets the next item in the path returning the element and the remaining @@ -78,17 +77,17 @@ end -- Finds the grammar node for a fragment in a given grammar. local function extract_grammar_node(grammar, fragment) - assert(grammar.type ~= "scalar") - local errmsg = "Invalid path: "..fragment.name - if grammar.type == "table" then + local handlers = {} + function handlers.struct () return grammar.members[fragment.name] end + function handlers.table () if grammar.keys[fragment.name] == nil then - return assert(grammar.values[fragment.name], errmsg) + return grammar.values[fragment.name] else return grammar.keys[fragment.name] end - else - return assert(grammar[fragment.name], errmsg) end + local errmsg = "Invalid path:"..fragment.name + return assert(assert(handlers[grammar.type](), errmsg), errmsg) end -- Converts an XPath path to a lua array consisting of path componants. @@ -98,75 +97,59 @@ function convert_path(grammar, path) local node = grammar for element in split_path(path) do node = extract_grammar_node(node, element) - local luapath, next_node = handle(node.type, element, node) - luapath.grammar = node + local luapath = handle(node, element) table.insert(ret, luapath) - node = next_node + --node = next_node end return ret end -- Returns a resolver for a paticular schema and *lua* path. -function resolver(schema, path) - local grammar = datalib.data_grammar_from_schema(schema) +function resolver(path) local handlers = {} - local function handle(scm, prod, data, path) + local function handle(data, path) if #path == 0 then return data end - return assert(handlers[scm.kind], scm.kind)(scm, prod, data, path) - end - function handlers.list(scm, prod, data, path) local head = table.remove(path, 1) + local handler = assert(handlers[head.grammar.type], head.grammar.type) + return handler(head, data, path) + end + function handlers.table(head, data, path) -- The list can either be a ctable or a plain old lua table, if it's the -- ctable then it requires some more work to retrive the data. - local prod = prod.members[head.name] local data = data[normalize_id(head.name)] - if #prod.keys > 1 then error("Multiple key values are not supported!") end - if prod.key_ctype then + if #head.grammar.keys > 1 then + error("Multiple key values are not supported!") + end + if head.grammar.key_ctype then -- It's a ctable and we need to prepare the key so we can lookup the -- pointer to the entry and then convert the values back to lua. - local kparser = valuelib.types[scm.body[scm.key].primitive_type].parse - local key = kparser(head.keys[scm.key]) - local ckey = ffi.new(prod.key_ctype) - ckey[scm.key] = key + local keyname = next(head.keys) + local ptype = head.grammar.keys[keyname].argument_type.primitive_type + local kparser = valuelib.types[ptype].parse + local key = kparser(head.keys[keyname]) + local ckey = ffi.new(head.grammar.key_ctype) + ckey[keyname] = key data = data:lookup_ptr(ckey).value else - data = data[normalize_id(head.keys[scm.key])] + data = data[normalize_id(head.keys[next(head.keys)])] end - - if #path == 0 then return data end - local peek = path[1] - scm = scm.body[peek.name] - prod = prod.values[peek.name] - return handle(scm, prod, data, path) + return handle(data, path) end - function handlers.container(scm, prod, data, path) - local head = table.remove(path, 1) - prod = prod.members[head.name] + function handlers.struct(head, data, path) data = data[normalize_id(head.name)] - if #path == 0 then return data end - local peek = path[1] - scm = scm.body[peek.name] - return handle(scm, prod, data, path) + return handle(data, path) end - handlers["leaf-list"] = function (scm, prod, data, path) - local head = table.remove(path, 1) - if #path ~= 0 then error("Paths can't go beyond leaf-lists.") end - return data[normalize_id(head.name)][head.key] + function handlers.array(head, data, path) + data = data[normalize_id(head.name)][head.key] + return handle(data, path) end - function handlers.leaf(scm, prod, data, path) - local head = table.remove(path, 1) + function handlers.scalar(head, data, path) if #path ~= 0 then error("Paths can't go beyond leaves.") end return data[normalize_id(head.name)] end - function handlers.module(scm, prod, data, path) - local peek = path[1] - scm = scm.body[peek.name] - return handle(scm, prod, data, path) - end return function (data) - local data = lib.deepcopy(data) - return handle(schema, grammar, data, path) + return handle(data, path) end end @@ -203,7 +186,7 @@ function selftest() local grammar = datalib.data_grammar_from_schema(scm) -- Test path to lua path. - local path = convert_path(grammar.members,"/routes/route[addr=1.2.3.4]/port") + local path = convert_path(grammar,"/routes/route[addr=1.2.3.4]/port") assert(path[1].name == "routes") assert(path[2].name == "route") @@ -211,7 +194,7 @@ function selftest() assert(path[2].keys["addr"] == "1.2.3.4") assert(path[3].name == "port") - local path = convert_path(grammar.members, "/blocked-ips[position()=4]/") + local path = convert_path(grammar, "/blocked-ips[position()=4]/") assert(path[1].name == "blocked-ips") assert(path[1].key == 4) @@ -232,15 +215,14 @@ function selftest() local data = datalib.load_data_for_schema(scm, data_src) -- Try resolving a path in a list (ctable). - local path = convert_path(grammar.members,"/routes/route[addr=1.2.3.4]/port") + local path = convert_path(grammar,"/routes/route[addr=1.2.3.4]/port") assert(resolver(scm, path)(data) == 2) - local path = convert_path(grammar.members, - "/routes/route[addr=255.255.255.255]/port") + local path = convert_path(grammar,"/routes/route[addr=255.255.255.255]/port") assert(resolver(scm, path)(data) == 7) -- Try resolving a leaf-list - local path = convert_path(grammar.members,"/blocked-ips[position()=1]") + local path = convert_path(grammar,"/blocked-ips[position()=1]") assert(resolver(scm, path)(data) == util.ipv4_pton("8.8.8.8")) -- Try resolving a path for a list (non-ctable) @@ -270,11 +252,9 @@ function selftest() local fruit_prod = datalib.data_grammar_from_schema(fruit_scm) local fruit_data = datalib.load_data_for_schema(fruit_scm, fruit_data_src) - local path = convert_path(fruit_prod.members, - "/bowl/fruit[name=banana]/rating") + local path = convert_path(fruit_prod, "/bowl/fruit[name=banana]/rating") assert(resolver(fruit_scm, path)(fruit_data) == 10) - local path = convert_path(fruit_prod.members, - "/bowl/fruit[name=apple]/rating") + local path = convert_path(fruit_prod, "/bowl/fruit[name=apple]/rating") assert(resolver(fruit_scm, path)(fruit_data) == 6) end From f35af3ab210afe08d00992ee787ec55dfc8df794 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 22 Nov 2016 10:48:54 +0100 Subject: [PATCH 278/631] Fix a bug and use string:split() in lib.yang.xpath --- src/lib/yang/xpath.lua | 54 +++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/xpath.lua index a8e0d4443d..cd830cfb37 100644 --- a/src/lib/yang/xpath.lua +++ b/src/lib/yang/xpath.lua @@ -43,10 +43,9 @@ end local handlers = {} function handlers.scalar(grammar, fragment) - return {name=fragment.name, grammar=grammar} -end + return {name=fragment.name, grammar=grammar} end function handlers.struct(grammar, fragment) - return {name=fragment.name, grammar=grammar} + return {name=fragment.name, keys=fragment.query, grammar=grammar} end function handlers.table(grammar, fragment) return {name=fragment.name, keys=fragment.query, grammar=grammar} @@ -59,35 +58,18 @@ function handle(grammar, fragment) return assert(handlers[grammar.type], grammar.type)(grammar, fragment) end --- Gets the next item in the path returning the element and the remaining --- path fragment. For example "router.routes.route" will return "router" --- and "routes.route". If the end is reached it'll return nil. -local function next_element(path) - return string.match(path, "([^/]+)/?(.*)") -end - -local function split_path(path) - local tail = path - return function () - local head - head, tail = next_element(tail) - if head == nil then return head else return extract_parts(head) end - end -end - -- Finds the grammar node for a fragment in a given grammar. -local function extract_grammar_node(grammar, fragment) +local function extract_grammar_node(grammar, name) local handlers = {} - function handlers.struct () return grammar.members[fragment.name] end + function handlers.struct () return grammar.members[name] end function handlers.table () - if grammar.keys[fragment.name] == nil then - return grammar.values[fragment.name] + if grammar.keys[name] == nil then + return grammar.values[name] else - return grammar.keys[fragment.name] + return grammar.keys[name] end end - local errmsg = "Invalid path:"..fragment.name - return assert(assert(handlers[grammar.type](), errmsg), errmsg) + return assert(assert(handlers[grammar.type], grammar.type)(), name) end -- Converts an XPath path to a lua array consisting of path componants. @@ -95,11 +77,13 @@ end function convert_path(grammar, path) local ret = {} local node = grammar - for element in split_path(path) do - node = extract_grammar_node(node, element) - local luapath = handle(node, element) + if path:sub(1, 1) == "/" then path = path:sub(2) end -- remove leading / + if path:sub(-1) == "/" then path = path:sub(1, -2) end -- remove trailing / + for element in path:split("/") do + local parts = extract_parts(element) + node = extract_grammar_node(node, parts.name) + local luapath = handle(node, parts) table.insert(ret, luapath) - --node = next_node end return ret end @@ -216,14 +200,14 @@ function selftest() -- Try resolving a path in a list (ctable). local path = convert_path(grammar,"/routes/route[addr=1.2.3.4]/port") - assert(resolver(scm, path)(data) == 2) + assert(resolver(path)(data) == 2) local path = convert_path(grammar,"/routes/route[addr=255.255.255.255]/port") - assert(resolver(scm, path)(data) == 7) + assert(resolver(path)(data) == 7) -- Try resolving a leaf-list local path = convert_path(grammar,"/blocked-ips[position()=1]") - assert(resolver(scm, path)(data) == util.ipv4_pton("8.8.8.8")) + assert(resolver(path)(data) == util.ipv4_pton("8.8.8.8")) -- Try resolving a path for a list (non-ctable) local fruit_schema_src = [[module fruit-bowl { @@ -253,8 +237,8 @@ function selftest() local fruit_data = datalib.load_data_for_schema(fruit_scm, fruit_data_src) local path = convert_path(fruit_prod, "/bowl/fruit[name=banana]/rating") - assert(resolver(fruit_scm, path)(fruit_data) == 10) + assert(resolver(path)(fruit_data) == 10) local path = convert_path(fruit_prod, "/bowl/fruit[name=apple]/rating") - assert(resolver(fruit_scm, path)(fruit_data) == 6) + assert(resolver(path)(fruit_data) == 6) end From 8bcf5207d88eabe4f148f6483f1ad6af7bd25d2b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 10:14:09 +0100 Subject: [PATCH 279/631] Rename xpath.lua to path.lua --- src/lib/yang/{xpath.lua => path.lua} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/lib/yang/{xpath.lua => path.lua} (100%) diff --git a/src/lib/yang/xpath.lua b/src/lib/yang/path.lua similarity index 100% rename from src/lib/yang/xpath.lua rename to src/lib/yang/path.lua From 42faa3402803fe791e53203daeddb74e038ecec6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 10:17:12 +0100 Subject: [PATCH 280/631] Localize handlers for path.lua:convert_path --- src/lib/yang/path.lua | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index cd830cfb37..4433e612a0 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -41,23 +41,6 @@ local function extract_parts(fragment) return rtn end -local handlers = {} -function handlers.scalar(grammar, fragment) - return {name=fragment.name, grammar=grammar} end -function handlers.struct(grammar, fragment) - return {name=fragment.name, keys=fragment.query, grammar=grammar} -end -function handlers.table(grammar, fragment) - return {name=fragment.name, keys=fragment.query, grammar=grammar} -end -function handlers.array(grammar, fragment) - local position = fragment.query["position()"] - return {name=fragment.name, key=tonumber(position), grammar=grammar} -end -function handle(grammar, fragment) - return assert(handlers[grammar.type], grammar.type)(grammar, fragment) -end - -- Finds the grammar node for a fragment in a given grammar. local function extract_grammar_node(grammar, name) local handlers = {} @@ -75,6 +58,24 @@ end -- Converts an XPath path to a lua array consisting of path componants. -- A path component can then be resolved on a yang data tree: function convert_path(grammar, path) + local handlers = {} + function handlers.scalar(grammar, fragment) + return {name=fragment.name, grammar=grammar} + end + function handlers.struct(grammar, fragment) + return {name=fragment.name, grammar=grammar} + end + function handlers.table(grammar, fragment) + return {name=fragment.name, keys=fragment.query, grammar=grammar} + end + function handlers.array(grammar, fragment) + local position = fragment.query["position()"] + return {name=fragment.name, key=tonumber(position), grammar=grammar} + end + local function handle(grammar, fragment) + return assert(handlers[grammar.type], grammar.type)(grammar, fragment) + end + local ret = {} local node = grammar if path:sub(1, 1) == "/" then path = path:sub(2) end -- remove leading / From 5043daf84628865225145a50ac37df6b4002481b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 11:03:48 +0100 Subject: [PATCH 281/631] lib.yang.path test tweaks --- src/lib/yang/path.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 4433e612a0..99c72b68df 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -148,7 +148,7 @@ function load_from_path(path) end function selftest() - print("selftest: lib.yang.xpath") + print("selftest: lib.yang.path") local schema_src = [[module snabb-simple-router { namespace snabb:simple-router; prefix simple-router; @@ -242,4 +242,5 @@ function selftest() local path = convert_path(fruit_prod, "/bowl/fruit[name=apple]/rating") assert(resolver(path)(fruit_data) == 6) + print("selftest: ok") end From 8b3f9a381ca624a2366ee1e16ce0cfa96e697084 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 13:19:34 +0100 Subject: [PATCH 282/631] Precompute path resolvers before running Before, we would look at the grammar when resolving a path in data, to determine what to do at the next path item. Now we resolve what to do ahead of time, and just return a closure that does the lookup. Resolve also returns the grammar for the data that will be looked up, as a second value. --- src/lib/yang/path.lua | 173 ++++++++++++++++++++++++++++++------------ 1 file changed, 125 insertions(+), 48 deletions(-) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 99c72b68df..b40a9074a2 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -26,12 +26,19 @@ module(..., package.seeall) local ffi = require("ffi") +local equal = require("core.lib").equal local schemalib = require("lib.yang.schema") local datalib = require("lib.yang.data") local valuelib = require("lib.yang.value") local util = require("lib.yang.util") local normalize_id = datalib.normalize_id +local function table_keys(t) + local ret = {} + for k, v in pairs(t) do table.insert(ret, k) end + return ret +end + local function extract_parts(fragment) local rtn = {query={}} rtn.name = string.match(fragment, "([^%[]+)") @@ -89,53 +96,123 @@ function convert_path(grammar, path) return ret end +function parse_path(path) + local ret = {} + for element in path:split("/") do + if element ~= '' then table.insert(ret, extract_parts(element)) end + end + return ret +end + -- Returns a resolver for a paticular schema and *lua* path. -function resolver(path) - local handlers = {} - local function handle(data, path) - if #path == 0 then return data end - local head = table.remove(path, 1) - local handler = assert(handlers[head.grammar.type], head.grammar.type) - return handler(head, data, path) - end - function handlers.table(head, data, path) - -- The list can either be a ctable or a plain old lua table, if it's the - -- ctable then it requires some more work to retrive the data. - local data = data[normalize_id(head.name)] - if #head.grammar.keys > 1 then - error("Multiple key values are not supported!") +function resolver(grammar, path) + local function prepare_table_key(keys, ctype, query) + local static_key = ctype and datalib.typeof(ctype)() or {} + for k,_ in pairs(query) do + if not keys[k] then error("'"..key_name.."' is not a table key") end + end + for k,grammar in pairs(keys) do + local v = query[k] or grammar.default + if v == nil then + error("Table query missing required key '"..k.."'") + end + local key_primitive_type = grammar.argument_type.primitive_type + local parser = valuelib.types[key_primitive_type].parse + static_key[normalize_id(k)] = parser(v, 'path query value') + end + return static_key + end + local function ctable_getter(key, getter) + return function(data) + local data = getter(data):lookup_ptr(key) + if data == nil then error("Not found") end + return data.value end - if head.grammar.key_ctype then - -- It's a ctable and we need to prepare the key so we can lookup the - -- pointer to the entry and then convert the values back to lua. - local keyname = next(head.keys) - local ptype = head.grammar.keys[keyname].argument_type.primitive_type - local kparser = valuelib.types[ptype].parse - local key = kparser(head.keys[keyname]) - local ckey = ffi.new(head.grammar.key_ctype) - ckey[keyname] = key - - data = data:lookup_ptr(ckey).value + end + local function table_getter(key, getter) + return function(data) + local data = getter(data)[key] + if data == nil then error("Not found") end + return data + end + end + local function slow_table_getter(key, getter) + return function(data) + for k,v in pairs(getter(data)) do + if equal(k, key) then return v end + end + error("Not found") + end + end + local function compute_table_getter(grammar, key, getter) + if grammar.string_key then + return table_getter(key[normalize_id(grammar.string_key)], getter) + elseif grammar.key_ctype and grammar.value_ctype then + return ctable_getter(key, getter) + elseif grammar.key_ctype then + return table_getter(key, getter) else - data = data[normalize_id(head.keys[next(head.keys)])] + return slow_table_getter(key, getter) end - return handle(data, path) end - function handlers.struct(head, data, path) - data = data[normalize_id(head.name)] - return handle(data, path) + local function handle_table_query(grammar, query, getter) + local key = prepare_table_key(grammar.keys, grammar.key_ctype, query) + local child_grammar = {type="struct", members=grammar.values, + ctype=grammar.value_ctype} + local child_getter = compute_table_getter(grammar, key, getter) + return child_getter, child_grammar + end + local function handle_array_query(grammar, query, getter) + if not equal(table_keys(query), {"position()"}) then + error("Arrays can only be indexed by position.") + end + local idx = tonumber(query["position()"]) + if idx < 1 or idx ~= math.floor(idx) then + error("Arrays can only be indexed by positive integers.") + end + -- Pretend that array elements are scalars. + local child_grammar = {type="scalar", argument_type=grammar.element_type, + ctype=grammar.ctype} + local function child_getter(data) + local array = getter(data) + if idx > #array then error("Index out of bounds") end + return array[idx] + end + return child_getter, child_grammar end - function handlers.array(head, data, path) - data = data[normalize_id(head.name)][head.key] - return handle(data, path) + local function handle_query(grammar, query, getter) + if equal(table_keys(query), {}) then return getter, grammar end + if grammar.type == 'array' then + return handle_array_query(grammar, query, getter) + elseif grammar.type == 'table' then + return handle_table_query(grammar, query, getter) + else + error("Path query parameters only supported for structs and tables.") + end end - function handlers.scalar(head, data, path) - if #path ~= 0 then error("Paths can't go beyond leaves.") end - return data[normalize_id(head.name)] + local function compute_getter(grammar, name, query, getter) + local child_grammar = grammar.members[name] + if not child_grammar then + error("Struct has no field named '"..name.."'.") + end + local id = normalize_id(name) + local function child_getter(data) + local struct = getter(data) + local child = struct[id] + if child == nil then + error("Struct instance has no field named '"..name.."'.") + end + return child + end + return handle_query(child_grammar, query, child_getter) end - return function (data) - return handle(data, path) + local getter, grammar = function(data) return data end, grammar + for _, elt in ipairs(path) do + -- All non-leaves of the path tree must be structs. + if grammar.type ~= 'struct' then error("Invalid path.") end + getter, grammar = compute_getter(grammar, elt.name, elt.query, getter) end + return getter, grammar end -- Loads a module and converts the rest of the path. @@ -200,15 +277,15 @@ function selftest() local data = datalib.load_data_for_schema(scm, data_src) -- Try resolving a path in a list (ctable). - local path = convert_path(grammar,"/routes/route[addr=1.2.3.4]/port") - assert(resolver(path)(data) == 2) + local path = parse_path("/routes/route[addr=1.2.3.4]/port") + assert(resolver(grammar, path)(data) == 2) - local path = convert_path(grammar,"/routes/route[addr=255.255.255.255]/port") - assert(resolver(path)(data) == 7) + local path = parse_path("/routes/route[addr=255.255.255.255]/port") + assert(resolver(grammar, path)(data) == 7) -- Try resolving a leaf-list - local path = convert_path(grammar,"/blocked-ips[position()=1]") - assert(resolver(path)(data) == util.ipv4_pton("8.8.8.8")) + local path = parse_path("/blocked-ips[position()=1]") + assert(resolver(grammar, path)(data) == util.ipv4_pton("8.8.8.8")) -- Try resolving a path for a list (non-ctable) local fruit_schema_src = [[module fruit-bowl { @@ -237,10 +314,10 @@ function selftest() local fruit_prod = datalib.data_grammar_from_schema(fruit_scm) local fruit_data = datalib.load_data_for_schema(fruit_scm, fruit_data_src) - local path = convert_path(fruit_prod, "/bowl/fruit[name=banana]/rating") - assert(resolver(path)(fruit_data) == 10) + local path = parse_path("/bowl/fruit[name=banana]/rating") + assert(resolver(fruit_prod, path)(fruit_data) == 10) - local path = convert_path(fruit_prod, "/bowl/fruit[name=apple]/rating") - assert(resolver(path)(fruit_data) == 6) + local path = parse_path("/bowl/fruit[name=apple]/rating") + assert(resolver(fruit_prod, path)(fruit_data) == 6) print("selftest: ok") end From e6f4635d06a03884078daed777ab98a6c1aae33b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 14:05:41 +0100 Subject: [PATCH 283/631] Wire up snabb config get with paths The resolver takes a path string now. --- src/apps/config/leader.lua | 5 ++--- src/lib/yang/path.lua | 24 ++++++++++++------------ src/program/config/common.lua | 6 ++---- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 40c50057c2..66bd0bf220 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -7,6 +7,7 @@ local ffi = require("ffi") local yang = require("lib.yang.yang") local data = require("lib.yang.data") local rpc = require("lib.yang.rpc") +local path_resolver = require("lib.yang.path").resolver local app = require("core.app") local shm = require("core.shm") local app_graph = require("core.config") @@ -94,9 +95,7 @@ function Leader:rpc_describe (args) end local function path_getter_for_grammar(grammar, path) - -- Implement me :) - assert(path == '/') - return function(data) return data end, grammar + return path_resolver(grammar, path) end local function path_printer_for_grammar(grammar, path) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index b40a9074a2..82170965e3 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -105,7 +105,7 @@ function parse_path(path) end -- Returns a resolver for a paticular schema and *lua* path. -function resolver(grammar, path) +function resolver(grammar, path_string) local function prepare_table_key(keys, ctype, query) local static_key = ctype and datalib.typeof(ctype)() or {} for k,_ in pairs(query) do @@ -207,7 +207,7 @@ function resolver(grammar, path) return handle_query(child_grammar, query, child_getter) end local getter, grammar = function(data) return data end, grammar - for _, elt in ipairs(path) do + for _, elt in ipairs(parse_path(path_string)) do -- All non-leaves of the path tree must be structs. if grammar.type ~= 'struct' then error("Invalid path.") end getter, grammar = compute_getter(grammar, elt.name, elt.query, getter) @@ -277,15 +277,15 @@ function selftest() local data = datalib.load_data_for_schema(scm, data_src) -- Try resolving a path in a list (ctable). - local path = parse_path("/routes/route[addr=1.2.3.4]/port") - assert(resolver(grammar, path)(data) == 2) + local getter = resolver(grammar, "/routes/route[addr=1.2.3.4]/port") + assert(getter(data) == 2) - local path = parse_path("/routes/route[addr=255.255.255.255]/port") - assert(resolver(grammar, path)(data) == 7) + local getter = resolver(grammar, "/routes/route[addr=255.255.255.255]/port") + assert(getter(data) == 7) -- Try resolving a leaf-list - local path = parse_path("/blocked-ips[position()=1]") - assert(resolver(grammar, path)(data) == util.ipv4_pton("8.8.8.8")) + local getter = resolver(grammar, "/blocked-ips[position()=1]") + assert(getter(data) == util.ipv4_pton("8.8.8.8")) -- Try resolving a path for a list (non-ctable) local fruit_schema_src = [[module fruit-bowl { @@ -314,10 +314,10 @@ function selftest() local fruit_prod = datalib.data_grammar_from_schema(fruit_scm) local fruit_data = datalib.load_data_for_schema(fruit_scm, fruit_data_src) - local path = parse_path("/bowl/fruit[name=banana]/rating") - assert(resolver(fruit_prod, path)(fruit_data) == 10) + local getter = resolver(fruit_prod, "/bowl/fruit[name=banana]/rating") + assert(getter(fruit_data) == 10) - local path = parse_path("/bowl/fruit[name=apple]/rating") - assert(resolver(fruit_prod, path)(fruit_data) == 6) + local getter = resolver(fruit_prod, "/bowl/fruit[name=apple]/rating") + assert(getter(fruit_data) == 6) print("selftest: ok") end diff --git a/src/program/config/common.lua b/src/program/config/common.lua index c7a0ec2164..d2000c0907 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -60,10 +60,8 @@ function parse_command_line(args, opts) end if opts.with_path then if #args == 0 then err("missing path argument") end - local path = table.remove(args, 1) - -- Waiting on our XPath parsing library :) - if path ~= '/' then err("paths other than / currently unimplemented") end - ret.path = path + -- FIXME: Validate path? + ret.path = table.remove(args, 1) end if opts.with_value then local parser = data_parser(ret.schema_name, ret.path, opts.command) From 68a7ab54de3534397a55d32cf096c1df654c503c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 16:20:09 +0100 Subject: [PATCH 284/631] Rename parse_string to parse in lib.yang.parser Also rename decode_string to parse_string. Adapt all callers. --- src/lib/yang/data.lua | 9 ++++----- src/lib/yang/parser.lua | 14 +++++++------- src/lib/yang/schema.lua | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 95836c461a..cf8e633e94 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -2,8 +2,7 @@ -- COPYING. module(..., package.seeall) -local parse_string = require("lib.yang.parser").parse_string -local decode_string = require("lib.yang.parser").decode_string +local parser_mod = require("lib.yang.parser") local schema = require("lib.yang.schema") local util = require("lib.yang.util") local value = require("lib.yang.value") @@ -365,7 +364,7 @@ function data_parser_from_grammar(production) function top_parsers.struct(production) local parser = visit1('(top level)', production) return function(str, filename) - local node = {statements=parse_string(str, filename)} + local node = {statements=parser_mod.parse(str, filename)} return parser.finish(parser.parse(node, parser.init())) end end @@ -373,7 +372,7 @@ function data_parser_from_grammar(production) local members = visitn(production.members) return function(str, filename) local ret = {} - for _, node in ipairs(parse_string(str, filename)) do + for _, node in ipairs(parser_mod.parse(str, filename)) do local sub = assert(members[node.keyword], 'unrecognized rpc: '..node.keyword) local data = sub.finish(sub.parse(node, sub.init())) @@ -394,7 +393,7 @@ function data_parser_from_grammar(production) function top_parsers.scalar(production) local parse = value_parser(production.argument_type) return function(str, filename) - return parse(decode_string(str, filename)) + return parse(parser_mod.parse_string(str, filename)) end end return assert(top_parsers[production.type])(production) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index a29a2e753c..157433835b 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -273,7 +273,7 @@ function Parser:parse_statement() self:error("Unexpected character found") end -function decode_string(str, filename) +function parse_string(str, filename) local parser = Parser.new(str, filename) parser:skip_whitespace() local str = parser:parse_string() @@ -282,7 +282,7 @@ function decode_string(str, filename) return str end -function parse_string(str, filename) +function parse(str, filename) local parser = Parser.new(str, filename) return parser:parse_module() end @@ -291,7 +291,7 @@ function parse_file(filename) local file_in = assert(io.open(filename)) local contents = file_in:read("*a") file_in:close() - return parse_string(contents, filename) + return parse(contents, filename) end function selftest() @@ -339,7 +339,7 @@ function selftest() local function test_module(src, exp) - local result = strip_locs(parse_string(src)) + local result = strip_locs(parse(src)) if not lib.equal(result, exp) then pp(result) pp(exp) @@ -377,7 +377,7 @@ function selftest() test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type"}}}}) - parse_string(require('lib.yang.ietf_inet_types_yang')) - parse_string(require('lib.yang.ietf_yang_types_yang')) - parse_string(require('lib.yang.ietf_softwire_yang')) + parse(require('lib.yang.ietf_inet_types_yang')) + parse(require('lib.yang.ietf_yang_types_yang')) + parse(require('lib.yang.ietf_softwire_yang')) end diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 88ebc51e40..9018b178b6 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -788,7 +788,7 @@ local function primitivize(schema) end function parse_schema(src, filename) - return schema_from_ast(parser.parse_string(src, filename)) + return schema_from_ast(parser.parse(src, filename)) end function parse_schema_file(filename) return schema_from_ast(parser.parse_file(filename)) From 2d72f21f77ed697b3cd7a57428635295763cff62 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 17:33:51 +0100 Subject: [PATCH 285/631] Add ability to print and parse bare arrays and tables --- src/lib/yang/data.lua | 47 +++++++++++++++++++++++++++++------------ src/lib/yang/parser.lua | 24 +++++++++++++++++++++ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index cf8e633e94..c7a6f2a874 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -381,15 +381,24 @@ function data_parser_from_grammar(production) return ret end end - local function generic_top_parser(production) - local parser = top_parsers.struct({ - type=struct, members={[production.keyword] = production}}) + function top_parsers.array(production) + local parser = visit1('[bare array]', production) return function(str, filename) - return parser(str, filename)[production.keyword] + local out = parser.init() + for _,v in ipairs(parser_mod.parse_strings(str, filename)) do + out = parser.parse({keyword='[bare array]', argument=v}, out) + end + end + end + function top_parsers.table(production) + local parser = visit1('[bare table]', production) + return function(str, filename) + local out = parser.init() + for _,v in ipairs(parser_mod.parse_statement_lists(str, filename)) do + out = parser.parse({keyword='[bare table]', statements=v}, out) + end end end - top_parsers.array = generic_top_parser - top_parsers.table = generic_top_parser function top_parsers.scalar(production) local parse = value_parser(production.argument_type) return function(str, filename) @@ -498,6 +507,8 @@ function data_printer_from_grammar(production) end end end + -- As a special case, the table handler allows the keyword to be nil, + -- for printing tables at the top level without keywords. function handlers.table(keyword, production) local key_order, value_order = {}, {} for k,_ in pairs(production.keys) do table.insert(key_order, k) end @@ -509,7 +520,7 @@ function data_printer_from_grammar(production) if production.key_ctype and production.value_ctype then return function(data, file, indent) for entry in data:iterate() do - print_keyword(keyword, file, indent) + if keyword then print_keyword(keyword, file, indent) end file:write('{\n') print_key(entry.key, file, indent..' ') print_value(entry.value, file, indent..' ') @@ -520,7 +531,7 @@ function data_printer_from_grammar(production) local id = normalize_id(production.string_key) return function(data, file, indent) for key, value in pairs(data) do - print_keyword(keyword, file, indent) + if keyword then print_keyword(keyword, file, indent) end file:write('{\n') print_key({[id]=key}, file, indent..' ') print_value(value, file, indent..' ') @@ -530,7 +541,7 @@ function data_printer_from_grammar(production) elseif production.key_ctype then return function(data, file, indent) for key, value in cltable.pairs(data) do - print_keyword(keyword, file, indent) + if keyword then print_keyword(keyword, file, indent) end file:write('{\n') print_key(key, file, indent..' ') print_value(value, file, indent..' ') @@ -540,7 +551,7 @@ function data_printer_from_grammar(production) else return function(data, file, indent) for key, value in pairs(data) do - print_keyword(keyword, file, indent) + if keyword then print_keyword(keyword, file, indent) end file:write('{\n') print_key(key, file, indent..' ') print_value(value, file, indent..' ') @@ -582,15 +593,23 @@ function data_printer_from_grammar(production) return file:flush() end end - local function generic_top_printer(production) - local printer = body_printer({[production.keyword] = production}) + function top_printers.table(production) + local printer = handlers.table(nil, production) return function(data, file) printer(data, file, '') return file:flush() end end - top_printers.array = generic_top_printer - top_printers.table = generic_top_printer + function top_printers.array(production) + local serialize = value_serializer(production.element_type) + return function(data, file, indent) + for _,v in ipairs(data) do + file:write(serialize(v)) + file:write('\n') + end + return file:flush() + end + end function top_printers.scalar(production) local serialize = value_serializer(production.argument_type) return function(data, file) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 157433835b..39bb704616 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -282,6 +282,30 @@ function parse_string(str, filename) return str end +function parse_strings(str, filename) + local parser = Parser.new(str, filename) + local ret = {} + parser:skip_whitespace() + while not parser:is_eof() do + table.insert(ret, parser:parse_string()) + parser:skip_whitespace() + end + return ret +end + +function parse_statement_lists(str, filename) + local parser = Parser.new(str, filename) + local ret = {} + parser:skip_whitespace() + while not parser:is_eof() do + parser:consume("{") + table.insert(ret, self:parse_statement_list()) + parser:consume("}") + parser:skip_whitespace() + end + return ret +end + function parse(str, filename) local parser = Parser.new(str, filename) return parser:parse_module() From 2691648cb48dbda279a727a2015f9140b078e265 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 17:37:15 +0100 Subject: [PATCH 286/631] Address nits --- src/apps/config/leader.lua | 8 ++------ src/lib/yang/path.lua | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 66bd0bf220..5620765d3f 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -94,12 +94,8 @@ function Leader:rpc_describe (args) return { native_schema = self.schema_name } end -local function path_getter_for_grammar(grammar, path) - return path_resolver(grammar, path) -end - local function path_printer_for_grammar(grammar, path) - local getter, subgrammar = path_getter_for_grammar(grammar, path) + local getter, subgrammar = path_resolver(grammar, path) local printer = data.data_printer_from_grammar(subgrammar) return function(data, file) return printer(getter(data), file) @@ -132,7 +128,7 @@ local function load_schema_support(schema_name) end local function path_parser_for_grammar(grammar, path) - local getter, subgrammar = path_getter_for_grammar(grammar, path) + local getter, subgrammar = path_resolver(grammar, path) return data.data_parser_from_grammar(subgrammar) end diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 82170965e3..c0b3c3fd50 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -25,7 +25,6 @@ -- value as the value. module(..., package.seeall) -local ffi = require("ffi") local equal = require("core.lib").equal local schemalib = require("lib.yang.schema") local datalib = require("lib.yang.data") From 3a67a937f8f6bb393a475f477b25a2b24592bdc3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 18:25:08 +0100 Subject: [PATCH 287/631] Add normalize_path --- src/lib/yang/path.lua | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 82170965e3..25a375a596 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -104,6 +104,18 @@ function parse_path(path) return ret end +function normalize_path(path) + local ret = {} + for _,part in ipairs(parse_path(path)) do + local str = part.name + local keys = table_keys(part.query) + table.sort(keys) + for _,k in ipairs(keys) do str = str..'['..k..'='..part.query[k]..']' end + table.insert(ret, str) + end + return '/'..table.concat(ret, '/') +end + -- Returns a resolver for a paticular schema and *lua* path. function resolver(grammar, path_string) local function prepare_table_key(keys, ctype, query) @@ -319,5 +331,13 @@ function selftest() local getter = resolver(fruit_prod, "/bowl/fruit[name=apple]/rating") assert(getter(fruit_data) == 6) + + assert(normalize_path('') == '/') + assert(normalize_path('//') == '/') + assert(normalize_path('/') == '/') + assert(normalize_path('//foo//bar//') == '/foo/bar') + assert(normalize_path('//foo[b=1][c=2]//bar//') == '/foo[b=1][c=2]/bar') + assert(normalize_path('//foo[c=1][b=2]//bar//') == '/foo[b=2][c=1]/bar') + print("selftest: ok") end From f3f6b8be9e42922a2c25edd1f8a1eb4507938659 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 22 Nov 2016 18:25:25 +0100 Subject: [PATCH 288/631] "snabb config set" of paths with leader app support --- src/apps/config/leader.lua | 27 ++++++++++++++++++++------- src/program/config/common.lua | 20 ++++++++++++-------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 66bd0bf220..79d0ab4aea 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -4,10 +4,11 @@ module(...,package.seeall) local S = require("syscall") local ffi = require("ffi") +local lib = require("core.lib") local yang = require("lib.yang.yang") local data = require("lib.yang.data") local rpc = require("lib.yang.rpc") -local path_resolver = require("lib.yang.path").resolver +local path_mod = require("lib.yang.path") local app = require("core.app") local shm = require("core.shm") local app_graph = require("core.config") @@ -95,7 +96,7 @@ function Leader:rpc_describe (args) end local function path_getter_for_grammar(grammar, path) - return path_resolver(grammar, path) + return path_mod.resolver(grammar, path) end local function path_printer_for_grammar(grammar, path) @@ -145,10 +146,22 @@ local function path_parser_for_schema_by_name(schema_name, path) end local function path_setter_for_grammar(grammar, path) - -- Implement me :) - assert(path == "/") - return function(config, subconfig) - return subconfig + path = path_mod.normalize_path(path) + if path == "/" then + return function(config, subconfig) return subconfig end + elseif path:sub(#path) == ']' then + -- Path ends in a query; it must denote an array or table item. + error('Unimplemented') + else + local head, tail = lib.dirname(path), lib.basename(path) + local getter, grammar = path_mod.resolver(grammar, head) + assert(grammar.type == 'struct') + assert(grammar.members[tail]) + local tail_id = data.normalize_id(tail) + return function(config, subconfig) + getter(config)[tail_id] = subconfig + return config + end end end @@ -165,7 +178,7 @@ function Leader:rpc_set_config (args) local parser = path_parser_for_schema_by_name(args.schema, args.path) local setter = path_setter_for_schema_by_name(args.schema, args.path) local subconfig = parser(args.config) - local new_config = setter(self.configuration, subconfig) + local new_config = setter(self.current_configuration, subconfig) local new_app_graph = self.setup_fn(new_config) local support = load_schema_support(args.schema) local actions = support.compute_config_actions(self.current_app_graph, diff --git a/src/program/config/common.lua b/src/program/config/common.lua index d2000c0907..4667896abf 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -8,6 +8,7 @@ local shm = require("core.shm") local rpc = require("lib.yang.rpc") local yang = require("lib.yang.yang") local data = require("lib.yang.data") +local path_resolver = require("lib.yang.path").resolver function show_usage(command, status, err_msg) if err_msg then print('error: '..err_msg) end @@ -23,13 +24,16 @@ local parse_command_line_opts = { require_schema = { default=false } } +local function path_grammar(schema_name, path) + local schema = yang.load_schema_by_name(schema_name) + local grammar = data.data_grammar_from_schema(schema) + local getter, subgrammar = path_resolver(grammar, path) + return subgrammar +end + local function data_parser(schema_name, path, command) - -- Waiting on XPath library. - assert(path == "/") assert(command ~= 'add') - return function (str) - return data.load_data_for_schema_by_name(schema_name, str) - end + return data.data_parser_from_grammar(path_grammar(schema_name, path)) end function parse_command_line(args, opts) @@ -92,10 +96,10 @@ function open_socket_or_die(instance_id) end function serialize_config(config, schema_name, path, command) - assert(path == nil or path == "/") assert(command ~= 'add') - return yang.print_data_for_schema_by_name(schema_name, config, - yang.string_output_file()) + local grammar = path_grammar(schema_name, path) + local printer = data.data_printer_from_grammar(grammar) + return printer(config, yang.string_output_file()) end function send_message(socket, msg_str) From b1301aee7f7076c47741a56b4ce8b0b404987530 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 10:03:36 +0100 Subject: [PATCH 289/631] "snabb config set" on table/array items --- src/apps/config/leader.lua | 64 +++++++++++++++++++++++++++++++++----- src/lib/yang/path.lua | 54 ++++++++++++++++++-------------- 2 files changed, 87 insertions(+), 31 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index ced9dbcba9..d0ec2bee2b 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -145,20 +145,70 @@ local function path_setter_for_grammar(grammar, path) path = path_mod.normalize_path(path) if path == "/" then return function(config, subconfig) return subconfig end - elseif path:sub(#path) == ']' then - -- Path ends in a query; it must denote an array or table item. - error('Unimplemented') - else - local head, tail = lib.dirname(path), lib.basename(path) + end + local head, tail = lib.dirname(path), lib.basename(path) + local tail_path = path_mod.parse_path(tail) + local tail_name, query = tail_path[1].name, tail_path[1].query + if lib.equal(query, {}) then + -- No query; the simple case. local getter, grammar = path_mod.resolver(grammar, head) assert(grammar.type == 'struct') - assert(grammar.members[tail]) - local tail_id = data.normalize_id(tail) + local tail_id = data.normalize_id(tail_name) return function(config, subconfig) getter(config)[tail_id] = subconfig return config end end + + -- Otherwise the path ends in a query; it must denote an array or + -- table item. + local getter, grammar = path_mod.resolver(grammar, head..'/'..tail_name) + if grammar.type == 'array' then + local idx = path_mod.prepare_array_lookup(query) + return function(config, subconfig) + local array = getter(config) + assert(idx <= #array) + array[idx] = subconfig + return config + end + elseif grammar.type == 'table' then + local key = path_mod.prepare_table_lookup(grammar.keys, + grammar.key_ctype, query) + if grammar.string_key then + key = key[normalize_id(grammar.string_key)] + return function(config, subconfig) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = subconfig + return config + end + elseif grammar.key_ctype and grammar.value_ctype then + return function(config, subconfig) + getter(config):update(key, subconfig) + return config + end + elseif grammar.key_ctype then + return function(config, subconfig) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = subconfig + return config + end + else + return function(config, subconfig) + local tab = getter(config) + for k,v in pairs(tab) do + if lib.equal(k, key) then + tab[k] = subconfig + return conig + end + end + error("Not found") + end + end + else + error('Query parameters only allowed on arrays and tables') + end end local function path_setter_for_schema(schema, path) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index f52a2c2715..466ea0737e 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -115,24 +115,36 @@ function normalize_path(path) return '/'..table.concat(ret, '/') end --- Returns a resolver for a paticular schema and *lua* path. -function resolver(grammar, path_string) - local function prepare_table_key(keys, ctype, query) - local static_key = ctype and datalib.typeof(ctype)() or {} - for k,_ in pairs(query) do - if not keys[k] then error("'"..key_name.."' is not a table key") end - end - for k,grammar in pairs(keys) do - local v = query[k] or grammar.default - if v == nil then - error("Table query missing required key '"..k.."'") - end - local key_primitive_type = grammar.argument_type.primitive_type - local parser = valuelib.types[key_primitive_type].parse - static_key[normalize_id(k)] = parser(v, 'path query value') +function prepare_array_lookup(query) + if not equal(table_keys(query), {"position()"}) then + error("Arrays can only be indexed by position.") + end + local idx = tonumber(query["position()"]) + if idx < 1 or idx ~= math.floor(idx) then + error("Arrays can only be indexed by positive integers.") + end + return idx +end + +function prepare_table_lookup(keys, ctype, query) + local static_key = ctype and datalib.typeof(ctype)() or {} + for k,_ in pairs(query) do + if not keys[k] then error("'"..key_name.."' is not a table key") end + end + for k,grammar in pairs(keys) do + local v = query[k] or grammar.default + if v == nil then + error("Table query missing required key '"..k.."'") end - return static_key + local key_primitive_type = grammar.argument_type.primitive_type + local parser = valuelib.types[key_primitive_type].parse + static_key[normalize_id(k)] = parser(v, 'path query value') end + return static_key +end + +-- Returns a resolver for a paticular schema and *lua* path. +function resolver(grammar, path_string) local function ctable_getter(key, getter) return function(data) local data = getter(data):lookup_ptr(key) @@ -167,20 +179,14 @@ function resolver(grammar, path_string) end end local function handle_table_query(grammar, query, getter) - local key = prepare_table_key(grammar.keys, grammar.key_ctype, query) + local key = prepare_table_lookup(grammar.keys, grammar.key_ctype, query) local child_grammar = {type="struct", members=grammar.values, ctype=grammar.value_ctype} local child_getter = compute_table_getter(grammar, key, getter) return child_getter, child_grammar end local function handle_array_query(grammar, query, getter) - if not equal(table_keys(query), {"position()"}) then - error("Arrays can only be indexed by position.") - end - local idx = tonumber(query["position()"]) - if idx < 1 or idx ~= math.floor(idx) then - error("Arrays can only be indexed by positive integers.") - end + local idx = prepare_array_lookup(query) -- Pretend that array elements are scalars. local child_grammar = {type="scalar", argument_type=grammar.element_type, ctype=grammar.ctype} From 21d1f7d4828956d833325276d5963951479ae6d2 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 10:16:28 +0100 Subject: [PATCH 290/631] Fix bug parsing bare arrays and tables --- src/lib/yang/data.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index c7a6f2a874..dbd92bd83d 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -388,6 +388,7 @@ function data_parser_from_grammar(production) for _,v in ipairs(parser_mod.parse_strings(str, filename)) do out = parser.parse({keyword='[bare array]', argument=v}, out) end + return parser.finish(out) end end function top_parsers.table(production) @@ -397,6 +398,7 @@ function data_parser_from_grammar(production) for _,v in ipairs(parser_mod.parse_statement_lists(str, filename)) do out = parser.parse({keyword='[bare table]', statements=v}, out) end + return parser.finish(out) end end function top_parsers.scalar(production) From 2eeade4f9f0b4082ddd3d09a6fdebda7542b958d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 10:38:07 +0100 Subject: [PATCH 291/631] Prepare for add and remove "snabb config" commands --- src/apps/config/leader.lua | 40 ++++++++++++++++++++++++----------- src/program/config/common.lua | 2 -- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index d0ec2bee2b..a85d23e649 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -142,7 +142,6 @@ local function path_parser_for_schema_by_name(schema_name, path) end local function path_setter_for_grammar(grammar, path) - path = path_mod.normalize_path(path) if path == "/" then return function(config, subconfig) return subconfig end end @@ -215,27 +214,44 @@ local function path_setter_for_schema(schema, path) return path_setter_for_grammar(data.data_grammar_from_schema(schema), path) end -local function path_setter_for_schema_by_name(schema_name, path) +function compute_set_config_fn (schema_name, path) return path_setter_for_schema(yang.load_schema_by_name(schema_name), path) end -function Leader:rpc_set_config (args) - assert(args.schema == self.schema_name) - local parser = path_parser_for_schema_by_name(args.schema, args.path) - local setter = path_setter_for_schema_by_name(args.schema, args.path) - local subconfig = parser(args.config) - local new_config = setter(self.current_configuration, subconfig) +function Leader:update_configuration (schema_name, update_fn, verb, path, arg) + assert(schema_name == self.schema_name) + local new_config = update_fn(self.current_configuration, arg) local new_app_graph = self.setup_fn(new_config) - local support = load_schema_support(args.schema) - local actions = support.compute_config_actions(self.current_app_graph, - new_app_graph, 'set', - args.path, subconfig) + local support = load_schema_support(schema_name) + local actions = support.compute_config_actions( + self.current_app_graph, new_app_graph, verb, path, arg) self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph self.current_configuration = new_config +end + +function Leader:handle_rpc_update_config (args, verb, compute_update_fn) + assert(args.schema == self.schema_name) + local path = path_mod.normalize_path(args.path) + local parser = path_parser_for_schema_by_name(args.schema, path) + self:update_configuration(args.schema, + compute_update_fn(args.schema, path), + 'set', path, parser(args.config)) return {} end +function Leader:rpc_set_config (args) + return self:handle_rpc_update_config(args, 'set', compute_set_config_fn) +end + +function Leader:rpc_add_config (args) + return self:handle_rpc_update_config(args, 'add', compute_add_config_fn) +end + +function Leader:rpc_remove_config (args) + return self:handle_rpc_update_config(args, 'remove', compute_remove_config_fn) +end + function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 4667896abf..f94602f42c 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -32,7 +32,6 @@ local function path_grammar(schema_name, path) end local function data_parser(schema_name, path, command) - assert(command ~= 'add') return data.data_parser_from_grammar(path_grammar(schema_name, path)) end @@ -96,7 +95,6 @@ function open_socket_or_die(instance_id) end function serialize_config(config, schema_name, path, command) - assert(command ~= 'add') local grammar = path_grammar(schema_name, path) local printer = data.data_printer_from_grammar(grammar) return printer(config, yang.string_output_file()) From ba5d3edbc56a8791b3c32baa998188b7bf697f7c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 12:21:29 +0100 Subject: [PATCH 292/631] Fix bug in lib.yang.parser.parse_statement_lists --- src/lib/yang/parser.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 39bb704616..bdd16091db 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -299,7 +299,7 @@ function parse_statement_lists(str, filename) parser:skip_whitespace() while not parser:is_eof() do parser:consume("{") - table.insert(ret, self:parse_statement_list()) + table.insert(ret, parser:parse_statement_list()) parser:consume("}") parser:skip_whitespace() end From 497a1d4ce3552238dc7837dd8aa66a121743dcab Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 12:21:44 +0100 Subject: [PATCH 293/631] Add support for add-config to leader --- src/apps/config/leader.lua | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index a85d23e649..589bde3b96 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -5,8 +5,10 @@ module(...,package.seeall) local S = require("syscall") local ffi = require("ffi") local lib = require("core.lib") +local cltable = require("lib.cltable") local yang = require("lib.yang.yang") local data = require("lib.yang.data") +local util = require("lib.yang.util") local rpc = require("lib.yang.rpc") local path_mod = require("lib.yang.path") local app = require("core.app") @@ -218,6 +220,86 @@ function compute_set_config_fn (schema_name, path) return path_setter_for_schema(yang.load_schema_by_name(schema_name), path) end +local function path_adder_for_grammar(grammar, path) + local top_grammar = grammar + local getter, grammar = path_mod.resolver(grammar, path) + if grammar.type == 'array' then + if grammar.ctype then + -- It's an FFI array; have to create a fresh one, sadly. + local setter = path_setter_for_grammar(top_grammar, path) + local elt_t = data.typeof(grammar.ctype) + local array_t = ffi.typeof('$[?]', elt_t) + return function(config, subconfig) + local cur = getter(config) + local new = array_t(#cur + #subconfig) + local i = 1 + for _,elt in ipairs(cur) do new[i-1] = elt; i = i + 1 end + for _,elt in ipairs(subconfig) do new[i-1] = elt; i = i + 1 end + return setter(config, util.ffi_array(new, elt_t)) + end + end + -- Otherwise we can add entries in place. + return function(config, subconfig) + local cur = getter(config) + for _,elt in ipairs(subconfig) do table.insert(cur, elt) end + return config + end + elseif grammar.type == 'table' then + -- Invariant: either all entries in the new subconfig are added, + -- or none are. + if grammar.key_ctype and grammar.value_ctype then + -- ctable. + return function(config, subconfig) + local ctab = getter(config) + for entry in subconfig:iterate() do + if ctab:lookup_ptr(entry.key) ~= nil then + error('already-existing entry', entry.key) + end + end + for entry in subconfig:iterate() do + ctab:add(entry.key, entry.value) + end + return config + end + elseif grammar.string_key or grammar.key_ctype then + -- cltable or string-keyed table. + local pairs = grammar.key_ctype and cltable.pairs or pairs + return function(config, subconfig) + local tab = getter(config) + for k,_ in pairs(subconfig) do + if tab[k] ~= nil then error('already-existing entry', k) end + end + for k,v in pairs(subconfig) do tab[k] = v end + return config + end + else + -- Sad quadratic loop. + return function(config, subconfig) + local tab = getter(config) + for key,val in pairs(tab) do + for k,_ in pairs(subconfig) do + if lib.equal(key, k) then + error('already-existing entry', key) + end + end + end + for k,v in pairs(subconfig) do tab[k] = v end + return config + end + end + else + error('Add only allowed on arrays and tables') + end +end + +local function path_adder_for_schema(schema, path) + return path_adder_for_grammar(data.data_grammar_from_schema(schema), path) +end + +function compute_add_config_fn (schema_name, path) + return path_adder_for_schema(yang.load_schema_by_name(schema_name), path) +end + function Leader:update_configuration (schema_name, update_fn, verb, path, arg) assert(schema_name == self.schema_name) local new_config = update_fn(self.current_configuration, arg) @@ -315,6 +397,8 @@ function Leader:handle_calls_from_peers() end end while peer.state == 'ready' do + -- Uncomment to get backtraces. + -- local success, reply = true, self:handle(peer.payload) local success, reply = pcall(self.handle, self, peer.payload) peer.payload = nil if success then From e3a0e7686e45425d36029fbd526633aa0540ed6e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 13:12:18 +0100 Subject: [PATCH 294/631] Add "snabb config remove" --- src/apps/config/leader.lua | 102 +++++++++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 9 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 589bde3b96..4d8bf1d531 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -121,7 +121,7 @@ function Leader:rpc_get_config (args) end local generic_schema_support = { - compute_config_actions = function(old_graph, new_graph, verb, path, subconf) + compute_config_actions = function(old_graph, new_graph, verb, path, ...) return app.compute_config_actions(old_graph, new_graph) end } @@ -201,7 +201,7 @@ local function path_setter_for_grammar(grammar, path) for k,v in pairs(tab) do if lib.equal(k, key) then tab[k] = subconfig - return conig + return config end end error("Not found") @@ -300,13 +300,92 @@ function compute_add_config_fn (schema_name, path) return path_adder_for_schema(yang.load_schema_by_name(schema_name), path) end -function Leader:update_configuration (schema_name, update_fn, verb, path, arg) +local function path_remover_for_grammar(grammar, path) + local top_grammar = grammar + local head, tail = lib.dirname(path), lib.basename(path) + local tail_path = path_mod.parse_path(tail) + local tail_name, query = tail_path[1].name, tail_path[1].query + local getter, grammar = path_mod.resolver(grammar, head..'/'..tail_name) + if grammar.type == 'array' then + if grammar.ctype then + -- It's an FFI array; have to create a fresh one, sadly. + local idx = path_mod.prepare_array_lookup(query) + local setter = path_setter_for_grammar(top_grammar, head) + local elt_t = data.typeof(grammar.ctype) + local array_t = ffi.typeof('$[?]', elt_t) + return function(config) + local cur = getter(config) + assert(i <= #cur) + local new = array_t(#cur - 1) + for i,elt in ipairs(cur) do + if i < idx then new[i-1] = elt end + if i > idx then new[i-2] = elt end + end + return setter(config, util.ffi_array(new, elt_t)) + end + end + -- Otherwise we can remove the entry in place. + return function(config) + local cur = getter(config) + assert(i <= #cur) + table.remove(cur, i) + return config + end + elseif grammar.type == 'table' then + local key = path_mod.prepare_table_lookup(grammar.keys, + grammar.key_ctype, query) + if grammar.string_key then + key = key[normalize_id(grammar.string_key)] + return function(config) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = nil + return config + end + elseif grammar.key_ctype and grammar.value_ctype then + return function(config) + getter(config):remove(key) + return config + end + elseif grammar.key_ctype then + return function(config) + local tab = getter(config) + assert(tab[key] ~= nil) + tab[key] = nil + return config + end + else + return function(config) + local tab = getter(config) + for k,v in pairs(tab) do + if lib.equal(k, key) then + tab[k] = nil + return config + end + end + error("Not found") + end + end + else + error('Remove only allowed on arrays and tables') + end +end + +local function path_remover_for_schema(schema, path) + return path_remover_for_grammar(data.data_grammar_from_schema(schema), path) +end + +function compute_remove_config_fn (schema_name, path) + return path_remover_for_schema(yang.load_schema_by_name(schema_name), path) +end + +function Leader:update_configuration (schema_name, update_fn, verb, path, ...) assert(schema_name == self.schema_name) - local new_config = update_fn(self.current_configuration, arg) + local new_config = update_fn(self.current_configuration, ...) local new_app_graph = self.setup_fn(new_config) local support = load_schema_support(schema_name) local actions = support.compute_config_actions( - self.current_app_graph, new_app_graph, verb, path, arg) + self.current_app_graph, new_app_graph, verb, path, ...) self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph self.current_configuration = new_config @@ -318,7 +397,7 @@ function Leader:handle_rpc_update_config (args, verb, compute_update_fn) local parser = path_parser_for_schema_by_name(args.schema, path) self:update_configuration(args.schema, compute_update_fn(args.schema, path), - 'set', path, parser(args.config)) + verb, path, parser(args.config)) return {} end @@ -331,7 +410,12 @@ function Leader:rpc_add_config (args) end function Leader:rpc_remove_config (args) - return self:handle_rpc_update_config(args, 'remove', compute_remove_config_fn) + assert(args.schema == self.schema_name) + local path = path_mod.normalize_path(args.path) + self:update_configuration(args.schema, + compute_remove_config_fn(args.schema, path), + 'remove', path) + return {} end function Leader:handle (payload) @@ -398,8 +482,8 @@ function Leader:handle_calls_from_peers() end while peer.state == 'ready' do -- Uncomment to get backtraces. - -- local success, reply = true, self:handle(peer.payload) - local success, reply = pcall(self.handle, self, peer.payload) + local success, reply = true, self:handle(peer.payload) + -- local success, reply = pcall(self.handle, self, peer.payload) peer.payload = nil if success then assert(type(reply) == 'string') From e3fabe7a0a08b085de70c228c2506df584bfec18 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 13:15:12 +0100 Subject: [PATCH 295/631] Re-enable pcall in leader --- src/apps/config/leader.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 4d8bf1d531..a91274a945 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -482,8 +482,8 @@ function Leader:handle_calls_from_peers() end while peer.state == 'ready' do -- Uncomment to get backtraces. - local success, reply = true, self:handle(peer.payload) - -- local success, reply = pcall(self.handle, self, peer.payload) + -- local success, reply = true, self:handle(peer.payload) + local success, reply = pcall(self.handle, self, peer.payload) peer.payload = nil if success then assert(type(reply) == 'string') From 9851ec82cd0656bb59986c8d357a266487143c8e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 15:33:00 +0100 Subject: [PATCH 296/631] Fix a link in program.config.README.md --- src/program/config/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/README.md b/src/program/config/README.md index 039cd9d7c8..c0d54e6812 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -23,7 +23,7 @@ include: * [`snabb config add`](./add/README_inc): augment configuration, for example by adding a routing table entry -* [`snabb config remove`](./delete/README_inc): remove a component from +* [`snabb config remove`](./remove/README_inc): remove a component from a configuration, for example removing a routing table entry * [`snabb config listen`](./listen/README_inc): provide an interface to From 1e32495c4d93fe008ba0186cd1257be69dc9386c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 23 Nov 2016 15:33:58 +0100 Subject: [PATCH 297/631] Fix more config README.md links --- src/program/config/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/program/config/README.md b/src/program/config/README.md index c0d54e6812..a9c5826590 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -12,21 +12,21 @@ the existing configuration of a Snabb instance. `snabb config` is a family of Snabb commands. Its sub-commands include: -* [`snabb config get`](./get/README_inc): read configuration data +* [`snabb config get`](./get/README): read configuration data -* [`snabb config get-state`](./get_state/README_inc): read state data +* [`snabb config get-state`](./get_state/README): read state data -* [`snabb config load`](./load/README_inc): load a new configuration +* [`snabb config load`](./load/README): load a new configuration -* [`snabb config set`](./set/README_inc): incrementally update configuration +* [`snabb config set`](./set/README): incrementally update configuration -* [`snabb config add`](./add/README_inc): augment configuration, for +* [`snabb config add`](./add/README): augment configuration, for example by adding a routing table entry -* [`snabb config remove`](./remove/README_inc): remove a component from +* [`snabb config remove`](./remove/README): remove a component from a configuration, for example removing a routing table entry -* [`snabb config listen`](./listen/README_inc): provide an interface to +* [`snabb config listen`](./listen/README): provide an interface to the `snabb config` functionality over a persistent socket, to minimize per-operation cost From 9c7687734013cbe8878164babdac47aaeeaff904 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 24 Nov 2016 19:01:01 +0100 Subject: [PATCH 298/631] First implementation of "snabb config listen" --- src/program/config/listen/README.inc | 15 ++ src/program/config/listen/listen.lua | 301 +++++++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 src/program/config/listen/README.inc create mode 100644 src/program/config/listen/listen.lua diff --git a/src/program/config/listen/README.inc b/src/program/config/listen/README.inc new file mode 100644 index 0000000000..f12ed113c4 --- /dev/null +++ b/src/program/config/listen/README.inc @@ -0,0 +1,15 @@ +Usage: + snabb config listen [OPTIONS] ID + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua new file mode 100644 index 0000000000..e01cebaf9c --- /dev/null +++ b/src/program/config/listen/listen.lua @@ -0,0 +1,301 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local S = require("syscall") +local ffi = require("ffi") +local rpc = require("lib.yang.rpc") +local yang = require("lib.yang.yang") +local data = require("lib.yang.data") +local path_lib = require("lib.yang.path") +local common = require("program.config.common") + +local function buffered_input_from_fd(fd) + local buf_size = 4096 + local buf = ffi.new('uint8_t[?]', buf_size) + local buf_end = 0 + local pos = 0 + local ret = {} + local eof = false + local function fill() + assert(pos == buf_end) + if eof then return 0 end + pos = 0 + buf_end = assert(S.read(fd, buf, buf_size)) + assert(0 <= buf_end and buf_end <= buf_size) + if buf_end == 0 then eof = true end + return buf_end + end + function ret:avail() return buf_end - pos end + function ret:getfd() return fd end + function ret:eof() return eof end + function ret:peek() + if pos == buf_end and fill() == 0 then return nil end + return string.char(buf[pos]) + end + function ret:discard() + assert(pos < buf_end) + pos = pos + 1 + end + return ret +end + +local whitespace_pat = '[ \n\r\t]' + +local function drop_buffered_whitespace(input) + while input:avail() > 0 and input:peek():match(whitespace_pat) do + input:discard() + end +end + +local function take_while(input, pat) + local out = {} + while input:peek() and input:peek():match(pat) do + table.insert(out, input.peek()) + input:discard() + end + return table.concat(out) +end + +local function check(input, ch) + if input:peek() ~= ch then return false end + input:discard() + return true +end + +local function consume(input, ch) + if not check(input, ch) then + if input:eof() then error('unexpected EOF') end + error('expected '..ch..', got '..input:peek()) + end +end + +local function consume_pat(input, pat) + local ch = input:peek() + if ch == nil then error('unexpected EOF') end + if not ch:match(pat) then error('unexpected character '..ch) end + input:discard() + return ch +end + +local function skip_whitespace(input) take_while(input, whitespace_pat) end + +-- Pattern describing characters that can appear literally in a JSON +-- string. +local literal_string_chars_pat = '%w' +do + -- Printable non-alphanumeric ASCII chars, excluding control + -- characters, backslash, and double-quote. + local punctuation = "!#$%&'()*+,-./:;<=>?@[]^_`{|}~ " + for i=1,#punctuation do + local punctuation_pat = '%'..punctuation:sub(i,i) + literal_string_chars_pat = literal_string_chars_pat..punctuation_pat + end + literal_string_chars_pat = '['..literal_string_chars_pat..']' +end +-- The escapable characters in JSON. +local escaped_string_chars = + { r="\r", n="\n", t="\t", ["\\"]="\\", ['"']='"', b="\b", f="\f", ["/"]="/" } + +local function read_json_string(input) + consume(input, '"') + local parts = {} + while not check(input, '"') do + -- JSON strings support unicode. The encoding of the JSON could + -- be anything though UTF-8 is the likely one. Assume the + -- encoding is ASCII-compatible (like UTF-8) and restrict + -- ourselves to printable ASCII characters. + local part = take_while(input, literal_string_chars_pat) + if part == '' then + consume(input, "\\") + for k,v in pairs(escaped_string_chars) do + if check(input, k) then part = v; break end + end + if part == '' and check(input, "u") then + -- 4-hex-digit unicode escape. We only support ASCII + -- tho. + local hex = '0x' + for i=1,4 do hex = hex..consume_pat(input, "%x") end + local code = assert(tonumber(hex)) + if code >= 128 then error('non-ASCII character: \\u00'..hex) end + part = string.char(code) + end + end + table.insert(parts, part) + end + return table.concat(parts) +end + +local function read_json_object(input) + consume(input, "{") + skip_whitespace(input) + local ret = {} + if not check(input, "}") then + repeat + skip_whitespace(input) + local k = read_json_string(input) + if ret[k] then error('duplicate key: '..k) end + skip_whitespace(input) + consume(input, ":") + skip_whitespace(input) + local v = read_json_string(input) + ret[k] = v + skip_whitespace(input) + until not check(input, ",") + skip_whitespace(input) + consume(input, "}") + end + return ret +end + +local function buffered_output() + local ret = { buf = {} } + function ret:write(str) table.insert(self.buf, str) end + function ret:flush_to_fd(fd) + local str = table.concat(self.buf) + local bytes = ffi.new('uint8_t[?]', #str, str) + local written = 0 + while written < #str do + local wrote = assert(S.write(fd, bytes + written, #str - written)) + written = written + wrote + end + end + return ret +end + +local function write_json_string(output, str) + output:write('"') + local pos = 1 + while pos <= #str do + local head = str:match('^('..literal_string_chars_pat..'+)', pos) + if head then + output:write(head) + pos = pos + #head + else + head = str:sub(pos, pos) + local escaped + for k,v in pairs(escaped_string_chars) do + if v == head then escaped = k; break end + end + if not escaped then + escaped = string.format("u00%.2x", head:byte(1)) + end + output:write('\\'..escaped) + pos = pos + 1 + end + end + output:write('"') +end + +local function write_json_object(output, obj) + output:write('{') + local comma = false + for k,v in pairs(obj) do + if comma then output:write(',') else comma = true end + write_json_string(output, k) + output:write(':') + write_json_string(output, v) + end + output:write('}') +end + +local request_handlers = {} +function request_handlers.get(schema_name, revision_date, path) + return {method='get-config', + args={schema=schema_name, revision=revision_date, path=path}} +end +function request_handlers.get_state(schema_name, revision_date, path) + return {method='get-state', + args={schema=schema_name, revision=revision_date, path=path}} +end +function request_handlers.set(schema_name, revision_date, path, value) + return {method='set-config', + args={schema=schema_name, revision=revision_date, path=path, + config=value}} +end +function request_handlers.add(schema_name, revision_date, path, value) + return {method='add-config', + args={schema=schema_name, revision=revision_date, path=path, + config=value}} +end +function request_handlers.remove(schema_name, revision_date, path, value) + return {method='remove-config', + args={schema=schema_name, revision=revision_date, path=path}} +end + +local function read_request(client, schema_name, revision_date) + local json = read_json_object(client) + local id, verb, path = assert(json.id), assert(json.verb), assert(json.path) + path = path_lib.normalize_path(path) + local handler = assert(request_handlers[data.normalize_id(verb)]) + local req = handler(schema_name, revision_date, path, json.value) + local function print_reply(reply) + local output = buffered_output() + write_json_object(output, {id=id, status='ok', value=reply.config}) + output:flush_to_fd(client:getfd()) + end + return req, print_reply +end + +function run(args) + args = common.parse_command_line(args, { command='listen' }) + local caller = rpc.prepare_caller('snabb-config-leader-v1') + local leader = common.open_socket_or_die(args.instance_id) + -- acquire leadership + local client = buffered_input_from_fd(0) -- stdin + local pollfds = S.types.t.pollfds({ + {fd=leader, events="in"}, + {fd=client, events="in"}}) + local pending_replies = {} + while true do + if client:avail() == 0 then + assert(S.poll(pollfds, -1)) + end + for _,pfd in ipairs(pollfds) do + if pfd.fd == leader:getfd() then + if pfd.ERR or pfd.HUP then + while #pending_replies > 0 do + local have_reply = table.remove(pending_replies) + have_reply(common.recv_message(leader)) + end + io.stderr:write('Leader hung up\n') + main.exit(1) + elseif pfd.IN then + assert(#pending_replies > 0) + local have_reply = table.remove(pending_replies) + have_reply(common.recv_message(leader)) + end + pfd.revents = 0 + elseif pfd.fd == client:getfd() then + if pfd.ERR or pfd.HUP or pfd.NVAL then + io.stderr:write('Client hung up\n') + main.exit(0) + end + if pfd.IN then + -- The JSON objects sent to us by the client can have + -- whitespace between them. Make sure we don't block + -- expecting a new datum when really it was just the + -- remote side sending whitespace. (Calling peek() + -- causes the buffer to fill, which itself shouldn't + -- block given the IN flag in the revents.) + client:peek() + drop_buffered_whitespace(client) + end + while client:avail() > 0 do + local request, print_reply = + read_request(client, args.schema_name, args.revision_date) + drop_buffered_whitespace(client) + local msg, parse_reply = rpc.prepare_call( + caller, request.method, request.args) + local function have_reply(msg) + return print_reply(parse_reply(msg)) + end + common.send_message(leader, msg) + table.insert(pending_replies, 1, have_reply) + end + pfd.revents = 0 + else + error('unreachable') + end + end + end +end From 5f102229c91d4da4a34080c14d75e903937d0a04 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 09:20:12 +0100 Subject: [PATCH 299/631] Leader doesn't hang up on client after one request --- src/apps/config/leader.lua | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index a91274a945..79069869c8 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -454,6 +454,13 @@ function Leader:handle_calls_from_peers() peer.state = 'error' peer.msg = 'length too long: '..peer.len end + elseif ch == '' then + if peer.len == 0 then + peer.state = 'done' + else + peer.state = 'error' + peer.msg = 'unexpected EOF' + end else peer.state = 'error' peer.msg = 'unexpected character: '..ch @@ -499,8 +506,9 @@ function Leader:handle_calls_from_peers() end while peer.state == 'reply' do if peer.pos == peer.len then - peer.state = 'done' - peer.buf, peer.len = nil, nil + peer.state = 'length' + peer.buf, peer.pos = nil, nil + peer.len = 0 else local count, err = peer.fd:write(peer.buf + peer.pos, peer.len - peer.pos) From 4d6cd7975f60f0962f39ae8f46fa298f5ee64ff5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 09:29:40 +0100 Subject: [PATCH 300/631] "snabb config listen" validates data --- src/program/config/add/add.lua | 2 +- src/program/config/common.lua | 6 +++--- src/program/config/listen/listen.lua | 16 +++++++++++++--- src/program/config/load/load.lua | 3 +-- src/program/config/set/set.lua | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/program/config/add/add.lua b/src/program/config/add/add.lua index cd6d4bb0f4..dcfcd6fe21 100644 --- a/src/program/config/add/add.lua +++ b/src/program/config/add/add.lua @@ -11,7 +11,7 @@ function run(args) { schema = args.schema_name, revision = args.revision_date, path = args.path, config = common.serialize_config( - args.value, args.schema_name, args.path, 'add') }) + args.value, args.schema_name, args.path) }) -- The reply is empty. main.exit(0) end diff --git a/src/program/config/common.lua b/src/program/config/common.lua index f94602f42c..055d69e22a 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -31,7 +31,7 @@ local function path_grammar(schema_name, path) return subgrammar end -local function data_parser(schema_name, path, command) +function data_parser(schema_name, path) return data.data_parser_from_grammar(path_grammar(schema_name, path)) end @@ -67,7 +67,7 @@ function parse_command_line(args, opts) ret.path = table.remove(args, 1) end if opts.with_value then - local parser = data_parser(ret.schema_name, ret.path, opts.command) + local parser = data_parser(ret.schema_name, ret.path) if #args == 0 then ret.value_str = io.stdin:read('*a') else @@ -94,7 +94,7 @@ function open_socket_or_die(instance_id) return socket end -function serialize_config(config, schema_name, path, command) +function serialize_config(config, schema_name, path) local grammar = path_grammar(schema_name, path) local printer = data.data_printer_from_grammar(grammar) return printer(config, yang.string_output_file()) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index e01cebaf9c..77baeb7304 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -198,6 +198,12 @@ local function write_json_object(output, obj) output:write('}') end +local function validate_value(schema_name, revision_date, path, value_str) + local parser = common.data_parser(schema_name, path) + local value = parser(value_str) + return common.serialize_config(value, schema_name, path) +end + local request_handlers = {} function request_handlers.get(schema_name, revision_date, path) return {method='get-config', @@ -208,16 +214,20 @@ function request_handlers.get_state(schema_name, revision_date, path) args={schema=schema_name, revision=revision_date, path=path}} end function request_handlers.set(schema_name, revision_date, path, value) + assert(value ~= nil) + local config = validate_value(schema_name, revision_date, path, value) return {method='set-config', args={schema=schema_name, revision=revision_date, path=path, - config=value}} + config=config}} end function request_handlers.add(schema_name, revision_date, path, value) + assert(value ~= nil) + local config = validate_value(schema_name, revision_date, path, value) return {method='add-config', args={schema=schema_name, revision=revision_date, path=path, - config=value}} + config=config}} end -function request_handlers.remove(schema_name, revision_date, path, value) +function request_handlers.remove(schema_name, revision_date, path) return {method='remove-config', args={schema=schema_name, revision=revision_date, path=path}} end diff --git a/src/program/config/load/load.lua b/src/program/config/load/load.lua index 38c4bad24d..9ef68d4c0b 100644 --- a/src/program/config/load/load.lua +++ b/src/program/config/load/load.lua @@ -9,8 +9,7 @@ function run(args) local response = common.call_leader( args.instance_id, 'set-config', { schema = args.schema_name, revision = args.revision_date, - config = common.serialize_config(args.config, args.schema_name, - 'load') }) + config = common.serialize_config(args.config, args.schema_name) }) -- The reply is empty. main.exit(0) end diff --git a/src/program/config/set/set.lua b/src/program/config/set/set.lua index 9ce0a583c6..7e5227a827 100644 --- a/src/program/config/set/set.lua +++ b/src/program/config/set/set.lua @@ -11,7 +11,7 @@ function run(args) { schema = args.schema_name, revision = args.revision_date, path = args.path, config = common.serialize_config( - args.value, args.schema_name, args.path, 'set') }) + args.value, args.schema_name, args.path) }) -- The reply is empty. main.exit(0) end From 362f2aee58ef85994834b6c29cb05f90d6404354 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 10:02:09 +0100 Subject: [PATCH 301/631] Listener prevents other clients from updating config --- src/apps/config/leader.lua | 20 +++++++++++++++----- src/lib/yang/snabb-config-leader-v1.yang | 7 +++++++ src/program/config/listen/listen.lua | 9 ++++++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 79069869c8..90663c1387 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -392,6 +392,9 @@ function Leader:update_configuration (schema_name, update_fn, verb, path, ...) end function Leader:handle_rpc_update_config (args, verb, compute_update_fn) + if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then + error('Attempt to modify configuration while listener attached') + end assert(args.schema == self.schema_name) local path = path_mod.normalize_path(args.path) local parser = path_parser_for_schema_by_name(args.schema, path) @@ -418,6 +421,13 @@ function Leader:rpc_remove_config (args) return {} end +function Leader:rpc_attach_listener (args) + assert(args.schema == self.schema_name) + if self.listen_peer ~= nil then error('Listener already attached') end + self.listen_peer = self.rpc_peer + return {} +end + function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end @@ -489,8 +499,10 @@ function Leader:handle_calls_from_peers() end while peer.state == 'ready' do -- Uncomment to get backtraces. + self.rpc_peer = peer -- local success, reply = true, self:handle(peer.payload) local success, reply = pcall(self.handle, self, peer.payload) + self.rpc_peer = nil peer.payload = nil if success then assert(type(reply) == 'string') @@ -525,13 +537,11 @@ function Leader:handle_calls_from_peers() end end end - if peer.state == 'done' then - peer.fd:close() - table.remove(peers, i) - elseif peer.state == 'error' then - print('error: '..peer.msg) + if peer.state == 'done' or peer.state == 'error' then + if peer.state == 'error' then print('error: '..peer.msg) end peer.fd:close() table.remove(peers, i) + if self.listen_peer == peer then self.listen_peer = nil end else i = i + 1 end diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index 340e076d32..9c0fb03e51 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -55,4 +55,11 @@ module snabb-config-leader-v1 { leaf path { type string; mandatory true; } } } + + rpc attach-listener { + input { + leaf schema { type string; mandatory true; } + leaf revision { type string; } + } + } } diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 77baeb7304..be2f16ed3a 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -246,11 +246,18 @@ local function read_request(client, schema_name, revision_date) return req, print_reply end +local function attach_listener(leader, caller, schema_name, revision_date) + local msg, parse_reply = rpc.prepare_call( + caller, 'attach-listener', {schema=schema_name, revision=revision_date}) + common.send_message(leader, msg) + return parse_reply(common.recv_message(leader)) +end + function run(args) args = common.parse_command_line(args, { command='listen' }) local caller = rpc.prepare_caller('snabb-config-leader-v1') local leader = common.open_socket_or_die(args.instance_id) - -- acquire leadership + attach_listener(leader, caller, args.schema_name, args.revision_date) local client = buffered_input_from_fd(0) -- stdin local pollfds = S.types.t.pollfds({ {fd=leader, events="in"}, From 2e28e71ae9a3ff472ca5222dd37b55ecc1716319 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 10:07:17 +0100 Subject: [PATCH 302/631] Move around READMEs --- src/program/config/add/README | 15 +++++++++++++++ src/program/config/add/README.inc | 16 +--------------- src/program/config/get/README | 15 +++++++++++++++ src/program/config/get/README.inc | 16 +--------------- src/program/config/get/get.lua | 2 +- src/program/config/listen/README | 15 +++++++++++++++ src/program/config/listen/README.inc | 16 +--------------- src/program/config/remove/README | 15 +++++++++++++++ src/program/config/remove/README.inc | 16 +--------------- src/program/config/set/README | 15 +++++++++++++++ src/program/config/set/README.inc | 16 +--------------- 11 files changed, 81 insertions(+), 76 deletions(-) create mode 100644 src/program/config/add/README mode change 100644 => 120000 src/program/config/add/README.inc create mode 100644 src/program/config/get/README mode change 100644 => 120000 src/program/config/get/README.inc create mode 100644 src/program/config/listen/README mode change 100644 => 120000 src/program/config/listen/README.inc create mode 100644 src/program/config/remove/README mode change 100644 => 120000 src/program/config/remove/README.inc create mode 100644 src/program/config/set/README mode change 100644 => 120000 src/program/config/set/README.inc diff --git a/src/program/config/add/README b/src/program/config/add/README new file mode 100644 index 0000000000..29eb9544f6 --- /dev/null +++ b/src/program/config/add/README @@ -0,0 +1,15 @@ +Usage: + snabb config add [OPTIONS] ID PATH [VALUE] + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/add/README.inc b/src/program/config/add/README.inc deleted file mode 100644 index 29eb9544f6..0000000000 --- a/src/program/config/add/README.inc +++ /dev/null @@ -1,15 +0,0 @@ -Usage: - snabb config add [OPTIONS] ID PATH [VALUE] - -Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA - - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help - diff --git a/src/program/config/add/README.inc b/src/program/config/add/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/add/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/get/README b/src/program/config/get/README new file mode 100644 index 0000000000..208310727e --- /dev/null +++ b/src/program/config/get/README @@ -0,0 +1,15 @@ +Usage: + snabb config get [OPTIONS] ID PATH + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/get/README.inc b/src/program/config/get/README.inc deleted file mode 100644 index 208310727e..0000000000 --- a/src/program/config/get/README.inc +++ /dev/null @@ -1,15 +0,0 @@ -Usage: - snabb config get [OPTIONS] ID PATH - -Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA - - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help - diff --git a/src/program/config/get/README.inc b/src/program/config/get/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/get/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/get/get.lua b/src/program/config/get/get.lua index 9bbe6571ed..0e23dc7209 100644 --- a/src/program/config/get/get.lua +++ b/src/program/config/get/get.lua @@ -4,7 +4,7 @@ module(..., package.seeall) local common = require("program.config.common") function run(args) - args = common.parse_command_line(args, { command='load', with_path=true }) + args = common.parse_command_line(args, { command='get', with_path=true }) local response = common.call_leader( args.instance_id, 'get-config', { schema = args.schema_name, revision = args.revision_date, diff --git a/src/program/config/listen/README b/src/program/config/listen/README new file mode 100644 index 0000000000..f12ed113c4 --- /dev/null +++ b/src/program/config/listen/README @@ -0,0 +1,15 @@ +Usage: + snabb config listen [OPTIONS] ID + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/listen/README.inc b/src/program/config/listen/README.inc deleted file mode 100644 index f12ed113c4..0000000000 --- a/src/program/config/listen/README.inc +++ /dev/null @@ -1,15 +0,0 @@ -Usage: - snabb config listen [OPTIONS] ID - -Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA - - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help - diff --git a/src/program/config/listen/README.inc b/src/program/config/listen/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/listen/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/remove/README b/src/program/config/remove/README new file mode 100644 index 0000000000..212726b901 --- /dev/null +++ b/src/program/config/remove/README @@ -0,0 +1,15 @@ +Usage: + snabb config remove [OPTIONS] ID PATH + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/remove/README.inc b/src/program/config/remove/README.inc deleted file mode 100644 index 212726b901..0000000000 --- a/src/program/config/remove/README.inc +++ /dev/null @@ -1,15 +0,0 @@ -Usage: - snabb config remove [OPTIONS] ID PATH - -Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA - - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help - diff --git a/src/program/config/remove/README.inc b/src/program/config/remove/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/remove/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/set/README b/src/program/config/set/README new file mode 100644 index 0000000000..cca7a54beb --- /dev/null +++ b/src/program/config/set/README @@ -0,0 +1,15 @@ +Usage: + snabb config set [OPTIONS] ID PATH [VALUE] + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/set/README.inc b/src/program/config/set/README.inc deleted file mode 100644 index cca7a54beb..0000000000 --- a/src/program/config/set/README.inc +++ /dev/null @@ -1,15 +0,0 @@ -Usage: - snabb config set [OPTIONS] ID PATH [VALUE] - -Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA - - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help - diff --git a/src/program/config/set/README.inc b/src/program/config/set/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/set/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file From 3fb219e5d2499abeb4394c243b93483ba833f7b7 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 10:20:05 +0100 Subject: [PATCH 303/631] Update snabb config usage files --- src/program/config/add/README | 21 +++++++++++---------- src/program/config/get/README | 21 ++++++++++----------- src/program/config/listen/README | 20 +++++++++----------- src/program/config/remove/README | 20 +++++++++----------- src/program/config/set/README | 21 +++++++++++---------- 5 files changed, 50 insertions(+), 53 deletions(-) diff --git a/src/program/config/add/README b/src/program/config/add/README index 29eb9544f6..7c3dcb60a2 100644 --- a/src/program/config/add/README +++ b/src/program/config/add/README @@ -1,15 +1,16 @@ -Usage: - snabb config add [OPTIONS] ID PATH [VALUE] +Usage: snabb config add [OPTION]... ID PATH [VALUE] +Add VALUE to the configuration of a Snabb network function. Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + -h, --help Display this message. - -r REVISION - --revision REVISION - --revision-date REVISION +If the --schema argument is not provided, "snabb config" will ask the +data plane for its native schema. - -h - --help +If VALUE is not given on the command line, it will be read from standard +input. +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. diff --git a/src/program/config/get/README b/src/program/config/get/README index 208310727e..6a4503baef 100644 --- a/src/program/config/get/README +++ b/src/program/config/get/README @@ -1,15 +1,14 @@ -Usage: - snabb config get [OPTIONS] ID PATH +Usage: snabb config get [OPTION]... ID PATH +Get the configuration for a Snabb network function. Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + -h, --help Display this message. - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help +If the --schema argument is not provided, "snabb config" will ask the +data plane for its native schema. The result will be printed on +standard output. +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. diff --git a/src/program/config/listen/README b/src/program/config/listen/README index f12ed113c4..f2d78464ff 100644 --- a/src/program/config/listen/README +++ b/src/program/config/listen/README @@ -1,15 +1,13 @@ -Usage: - snabb config listen [OPTIONS] ID +Usage: snabb config listen [OPTION]... ID +Open a persistent control channel to a Snabb network function. Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + -h, --help Display this message. - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help +If the --schema argument is not provided, "snabb config" will ask the +data plane for its native schema. +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. diff --git a/src/program/config/remove/README b/src/program/config/remove/README index 212726b901..6d6c740fff 100644 --- a/src/program/config/remove/README +++ b/src/program/config/remove/README @@ -1,15 +1,13 @@ -Usage: - snabb config remove [OPTIONS] ID PATH +Usage: snabb config remove [OPTION]... ID PATH +Remove an item from the configuration of a Snabb network function. Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + -h, --help Display this message. - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help +If the --schema argument is not provided, "snabb config" will ask the +data plane for its native schema. +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. diff --git a/src/program/config/set/README b/src/program/config/set/README index cca7a54beb..fc5430b42c 100644 --- a/src/program/config/set/README +++ b/src/program/config/set/README @@ -1,15 +1,16 @@ -Usage: - snabb config set [OPTIONS] ID PATH [VALUE] +Usage: snabb config set [OPTION]... ID PATH [VALUE] +Update the configuration for a Snabb network function. Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + -h, --help Display this message. - -r REVISION - --revision REVISION - --revision-date REVISION +If the --schema argument is not provided, "snabb config" will ask the +data plane for its native schema. - -h - --help +If VALUE is not given on the command line, it will be read from standard +input. +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. From 85a7dec0981149040b21e4b153e5b1848fdeaf19 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 10:25:46 +0100 Subject: [PATCH 304/631] Fix YANG parser bug for C comments --- src/lib/yang/parser.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index bdd16091db..97db2b7684 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -119,12 +119,13 @@ function Parser:skip_whitespace() -- Skip comments, which start with # and continue to the end of line. while self:check('/') do result = true - if self:check("*") then self:skip_c_comment() + if self:check("*") then + self:skip_c_comment() else self:consume("/") self:take_while('[^\n]') - self:take_while('%s') end + self:take_while('%s') end return result end @@ -392,6 +393,7 @@ function selftest() test_module("type;", {{keyword="type"}}) test_module("type string;", {{keyword="type", argument="string"}}) test_module("/** **/", {}) + test_module(" /** **/ ", {}) test_module("// foo bar;", {}) test_module("// foo bar;\nleaf port;", {{keyword="leaf", argument="port"}}) test_module("type/** hellooo */string;", {{keyword="type", argument="string"}}) From e15a812ec57fd69d0719d2666388f5a697250354 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 10:45:33 +0100 Subject: [PATCH 305/631] Listener writes output to stdout Before, we were writing to fd 0. I wonder how this worked! --- src/program/config/listen/listen.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index be2f16ed3a..96ed69f72d 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -241,7 +241,7 @@ local function read_request(client, schema_name, revision_date) local function print_reply(reply) local output = buffered_output() write_json_object(output, {id=id, status='ok', value=reply.config}) - output:flush_to_fd(client:getfd()) + output:flush_to_fd(1) -- stdout end return req, print_reply end From 4fa5f67318840681678fa7386e49fdd6986a38c4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 10:45:57 +0100 Subject: [PATCH 306/631] Update "snabb config" docs --- src/program/config/README.md | 120 ++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 24 deletions(-) diff --git a/src/program/config/README.md b/src/program/config/README.md index a9c5826590..89b06db0d4 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -1,11 +1,11 @@ # Data-plane configuration -Would be nice if you could update a Snabb program's configuration -while it's running, wouldn't it? Well never fear, `snabb config` is -here. Provided the data-plane author enabled this feature on their -side, users can run `snabb config` commands to query state or -configuration, provide a new configuration, or incrementally update -the existing configuration of a Snabb instance. +Wouldn't it be nice if you could update a Snabb program's configuration +while it's running? Well never fear, `snabb config` is here. Provided +the data-plane author enabled this feature on their side, users can run +`snabb config` commands to query state or configuration, provide a new +configuration, or incrementally update the existing configuration of a +Snabb instance. ## `snabb config` @@ -68,6 +68,8 @@ module snabb-simple-router { leaf active { type boolean; default true; } + leaf-list public-ip { type inet:ipv4-address; } + container routes { presence true; list route { @@ -94,6 +96,8 @@ routes { route { addr 1.2.3.4; port 1; } route { addr 2.3.4.5; port 2; } } +public-ip 10.10.10.10; +public-ip 10.10.10.11; ``` The surface syntax of data is the same as for YANG schemas; you can @@ -103,8 +107,8 @@ and the YANG schema quoting rules for strings apply. Note that containers like `route {}` only appear in the data syntax if they are marked as `presence true;` in the schema. -So indeed, `snabb config get ID /` -might print out just the output given above. +So indeed, `snabb config get ID /` might print out just the output given +above. Users can limit their query to a particular subtree via passing a different `PATH`. For example, with the same configuration, we can @@ -121,14 +125,11 @@ the schema, so the path components should reflect the data. A `list` is how YANG represents associations between keys and values. To query an element of a `list` item, use an XPath selector; for -example, to get the entry for IPv4 `1.2.3.4`, do: +example, to get the value associated with the key `1.2.3.4`, do: ``` $ snabb config get ID /routes/route[addr=1.2.3.4] -route { - addr 1.2.3.4; - port 1; -} +port 1; ``` Or to just get the port: @@ -144,6 +145,70 @@ Likewise, to change the port for `1.2.3.4`, do: $ snabb config set ID /routes/route[addr=1.2.3.4]/port 7 ``` +The general rule for paths and value syntax is that if a name appears in +the path, it won't appear in the value. Mostly this works as you would +expect, but there are a couple of edge cases for instances of `list` and +`leaf-list` nodes. For example: + +``` +$ snabb config get ID /routes/route[addr=1.2.3.4]/port +1 +$ snabb config get ID /routes/route[addr=1.2.3.4] +port 1; +$ snabb config get ID /routes/route +{ + addr 1.2.3.4; + port 1; +} +{ + addr 2.3.4.5; + port 2; +} +$ snabb config get ID /routes +route { + addr 1.2.3.4; + port 1; +} +route { + addr 2.3.4.5; + port 2; +} +``` + +Note the case when getting the `list` `/routes/route`: the syntax is a +sequence of brace-delimited entries. + +To select an entry from a `leaf-list`, use the `position()` selector: + +``` +$ snabb config get ID /public-ip[position()=1] +10.10.10.10 +$ snabb config get ID /public-ip[position()=2] +10.10.10.11 +``` + +As you can see, the indexes are 1-based. The syntax when getting or +setting a `leaf-list` directly by path is similar to the `list` case: a +sequence of whitespace-delimited bare values. + +``` +$ snabb config get ID /public-ip +10.10.10.10 +10.10.10.11 +$ snabb config set ID /public-ip "12.12.12.12 13.13.13.13" +$ snabb config get ID /public-ip +12.12.12.12 +13.13.13.13 +$ snabb config get ID / +active true; +routes { + route { addr 1.2.3.4; port 1; } + route { addr 2.3.4.5; port 2; } +} +public-ip 12.12.12.12; +public-ip 13.13.13.13; +``` + Values can be large, so it's also possible to take them from `stdin`. Do this by omitting the value: @@ -172,8 +237,8 @@ $ snabb config remove ID /routes/route[addr=1.2.3.4] One can of course augment a configuration as well: ``` -$ snabb config add ID /routes -route { +$ snabb config add ID /routes/route +{ addr 4.5.6.7; port 11; } @@ -181,9 +246,15 @@ route { ### Machine interface -The `listen` interface will support all of these operations with a -simple JSON protocol. Each request will be one JSON object with the -following properties: +The `listen` interface supports all of these operations with a simple +JSON protocol. `snabb config listen` reads JSON objects from `stdin`, +parses them, relays their action to the data plane, and writes responses +out to `stdout`. The requests are be processed in order, but +asynchronously; `snabb config listen` doesn't wait for a response from +the data plane before processing the next request. In this way, a +NETCONF agent can pipeline a number of requests. + +Each request is a JSON object with the following properties: - `id`: A request identifier; a string. Not used by `snabb config listen`; just a convenience for the other side. @@ -199,7 +270,8 @@ following properties: encoded as a string in the same syntax that the `snabb config set` accepts. -Each response from the server will also be one JSON object, with the following properties: +Each response from the server is also one JSON object, with the +following properties: - `id`: The identifier corresponding to the request. A string. @@ -218,7 +290,7 @@ $ snabb config listen -s snabb-simple-router ID { "id": "0", "verb": "get", "path": "/routes/route[addr=1.2.3.4]/port" } { "id": "1", "verb": "get", "path": "/routes/route[addr=2.3.4.5]/port" } { "id": "0", "status": "ok", "value: "1" } -{ "id": "1}, "status": "ok", "value: "2" } +{ "id": "1", "status": "ok", "value: "2" } ``` The above transcript indicates that requests may be pipelined: the @@ -227,10 +299,10 @@ waiting for responses. (For clarity, the first two JSON objects in the above transcript were entered by the user, in the console in this case; the second two are printed out by `snabb config` in response.) -The `snabb config listen` program will acquire exclusive write access -to the data plane, preventing other `snabb config` invocations from -modifying the configuration. In this way there is no need to provide -for notifications of changes made by other configuration clients. +The `snabb config listen` program acquires exclusive write access to the +data plane, preventing other `snabb config` invocations from modifying +the configuration. In this way there is no need to provide for +notifications of changes made by other configuration clients. ### Multiple schemas From 934122938e3d4731b8468a6022964618a396503f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 15:39:41 +0100 Subject: [PATCH 307/631] Add yang annotations to readme files --- src/apps/config/README.md | 6 +++--- src/program/config/README.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/apps/config/README.md b/src/apps/config/README.md index bdf976cb20..cb1d37dc8c 100644 --- a/src/apps/config/README.md +++ b/src/apps/config/README.md @@ -67,7 +67,7 @@ Snabb [textual data format for YANG data](../../lib/yang/README.md). For example the `snabb-config-leader-v1` schema supports a `get-config` RPC defined like this in the schema: -``` +```yang rpc get-config { input { leaf schema { type string; mandatory true; } @@ -82,7 +82,7 @@ rpc get-config { A request to this RPC might look like: -``` +```yang get-config { schema snabb-softwire-v1; path "/foo"; @@ -92,7 +92,7 @@ get-config { As you can see, non-mandatory inputs can be left out. A response might look like: -``` +```yang get-config { config "blah blah blah"; } diff --git a/src/program/config/README.md b/src/program/config/README.md index 89b06db0d4..fc726b190e 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -59,7 +59,7 @@ PIDs. Let's imagine that the configuration for the Snabb instance in question is modelled by the following YANG schema: -``` +```yang module snabb-simple-router { namespace snabb:simple-router; prefix simple-router; @@ -90,7 +90,7 @@ The configuration for a Snabb instance can be expressed in a text format that is derived from the schema. In this case it could look like: -``` +```yang active true; routes { route { addr 1.2.3.4; port 1; } From b013c2ea86f49136fef1541d3e7234893463a1cf Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 15:40:02 +0100 Subject: [PATCH 308/631] Fix bugs in config leader for tables --- src/apps/config/leader.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 90663c1387..df0efcb6b1 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -176,7 +176,7 @@ local function path_setter_for_grammar(grammar, path) local key = path_mod.prepare_table_lookup(grammar.keys, grammar.key_ctype, query) if grammar.string_key then - key = key[normalize_id(grammar.string_key)] + key = key[data.normalize_id(grammar.string_key)] return function(config, subconfig) local tab = getter(config) assert(tab[key] ~= nil) @@ -335,7 +335,7 @@ local function path_remover_for_grammar(grammar, path) local key = path_mod.prepare_table_lookup(grammar.keys, grammar.key_ctype, query) if grammar.string_key then - key = key[normalize_id(grammar.string_key)] + key = key[data.normalize_id(grammar.string_key)] return function(config) local tab = getter(config) assert(tab[key] ~= nil) From fcca73d6c83989a3aee35e617d5ca0f503d723d8 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 15:40:24 +0100 Subject: [PATCH 309/631] Add json read/write tests --- src/program/config/listen/listen.lua | 42 ++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 96ed69f72d..009c11a6cd 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -4,7 +4,6 @@ module(..., package.seeall) local S = require("syscall") local ffi = require("ffi") local rpc = require("lib.yang.rpc") -local yang = require("lib.yang.yang") local data = require("lib.yang.data") local path_lib = require("lib.yang.path") local common = require("program.config.common") @@ -50,7 +49,7 @@ end local function take_while(input, pat) local out = {} while input:peek() and input:peek():match(pat) do - table.insert(out, input.peek()) + table.insert(out, input:peek()) input:discard() end return table.concat(out) @@ -316,3 +315,42 @@ function run(args) end end end + +function selftest () + print('selftest: program.config.listen.listen') + local equal = require('core.lib').equal + local function test_json(str, obj) + local tmp = os.tmpname() + local f = io.open(tmp, 'w') + f:write(str) + f:write(" ") -- whitespace sentinel on the end. + f:close() + for i = 1,2 do + local fd = S.open(tmp, 'rdonly') + local input = buffered_input_from_fd(fd) + local parsed = read_json_object(input) + assert(equal(parsed, obj)) + assert(not input:eof()) + assert(check(input, " ")) + assert(not input:peek()) + assert(input:eof()) + fd:close() + + local fd = assert(S.open(tmp, 'wronly, trunc')) + local output = buffered_output() + write_json_object(output, parsed) + output:write(' ') -- sentinel + output:flush_to_fd(fd) + fd:close() + end + os.remove(tmp) + end + test_json('{}', {}) + test_json('{"foo":"bar"}', {foo='bar'}) + test_json('{"foo":"bar","baz":"qux"}', {foo='bar', baz='qux'}) + test_json('{ "foo" : "bar" , "baz" : "qux" }', + {foo='bar', baz='qux'}) + test_json('{ "fo\\u000ao" : "ba\\r " , "baz" : "qux" }', + {['fo\no']='ba\r ', baz='qux'}) + print('selftest: ok') +end From 69ac4794ed2a0ed3ee39237c4dec37761a39d0a3 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 15 Nov 2016 13:23:29 +0100 Subject: [PATCH 310/631] Add counters to yang schema --- src/lib/yang/snabb-softwire-v1.yang | 298 ++++++++++++++++++++- src/program/config/get_state/README.inc | 9 + src/program/config/get_state/get_state.lua | 134 +++++++++ 3 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 src/program/config/get_state/README.inc create mode 100644 src/program/config/get_state/get_state.lua diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index ab9046816e..af11c212ba 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -130,6 +130,8 @@ module snabb-softwire-v1 { } } + + container external-interface { description "Configuration for the external, internet-facing IPv4 @@ -377,4 +379,298 @@ module snabb-softwire-v1 { } } } -} + + container softwire-state { + description "State data about lwaftr."; + presence true; + config false; + + /* Decapsulation B4 queue */ + leaf in-ipv6-packets { + type yang:zero-based-counter64; + description "All valid outgoing IPv4 packets."; + } + + leaf drop-misplaced-not-ipv6-bytes { + type yang:zero-based-counter64; + description "Non-IPv6 packets incoming on IPv6 link."; + } + + leaf drop-unknown-protocol-ipv6-bytes { + type yang:zero-based-counter64; + description "Packets with an unknown IPv6 protocol."; + } + + leaf drop-in-by-policy-icmpv6-bytes { + type yang:zero-based-counter64; + description "Incoming ICMPv6 packets dropped because of current policy."; + } + + leaf out-icmpv4-packets { + type yang:zero-based-counter64; + description "Internally generated ICMPv4 packets."; + } + + leaf drop-too-big-type-but-not-code-icmpv6-bytes { + type yang:zero-based-counter64; + description + "Packet's ICMP type was 'Packet too big' but its ICMP code was not an + acceptable one for this type."; + } + + leaf drop-over-time-but-not-hop-limit-icmpv6-bytes { + type yang:zero-based-counter64; + description + "Packet's time limit was exceeded, but the hop limit was not."; + } + + /* Decapsulation internet queue */ + leaf drop-no-source-softwire-ipv6-bytes { + type yang:zero-based-counter64; + description + "No matching source softwire in the binding table; incremented whether + or not the reason was RFC7596."; + } + + leaf out-ipv4-packets { + type yang:zero-based-counter64; + description "Valid outgoing IPv4 packets."; + } + + leaf hairpin-ipv4-packets { + type yang:zero-based-counter64; + description "IPv4 packets going to a known b4 (hairpinned)."; + } + + leaf drop-out-by-policy-icmpv6-bytes { + type yang:zero-based-counter64; + description + "Internally generated ICMPv6 packets dropped because of current + policy."; + } + + leaf drop-over-rate-limit-icmpv6-bytes { + type yang:zero-based-counter64; + description + "Packets dropped because the outgoing ICMPv6 rate limit was reached."; + } + + leaf out-icmpv6-packets { + type yang:zero-based-counter64; + description "Internally generted ICMPv6 error packets."; + } + + /* Encapsulation internet queue */ + leaf in-ipv4-packets { + type yang:zero-based-counter64; + description "All valid outgoing IPv4 packets."; + } + + leaf drop-in-by-policy-icmpv4-bytes { + type yang:zero-based-counter64; + description "Incoming ICMPv4 packets dropped because of current policy."; + } + + leaf drop-misplaced-not-ipv4-bytes { + type yang:zero-based-counter64; + description "Non-IPv4 packets incoming on the IPv4 link."; + } + + leaf drop-bad-checksum-icmpv4-bytes { + type yang:zero-based-counter64; + description "ICMPv4 packets dropped because of a bad checksum."; + } + + leaf drop-all-ipv4-iface-bytes { + type yang:zero-based-counter64; + description + "All dropped packets and bytes that came in over IPv4 interfaces, + whether or not they actually IPv4 (they only include data about + packets that go in/out over the wires, excluding internally generated + ICMP packets)."; + } + + leaf drop-all-ipv6-iface-bytes { + type yang:zero-based-counter64; + description + "All dropped packets and bytes that came in over IPv6 interfaces, + whether or not they actually IPv6 (they only include data about packets + that go in/out over the wires, excluding internally generated ICMP + packets)."; + } + + /* Encapsulation B4 queue */ + leaf out-ipv6-packets { + type yang:zero-based-counter64; + description "All valid outgoing IPv6 packets."; + } + + leaf drop-over-mtu-but-dont-fragment-ipv4-bytes { + type yang:zero-based-counter64; + description + "IPv4 packets whose size exceeded the MTU, but the DF (Don't Fragment) + flag was set."; + } + + leaf drop-ttl-zero-ipv4-bytes { + type yang:zero-based-counter64; + description "IPv4 packets dropped because their TTL was zero."; + } + + leaf drop-out-by-policy-icmpv4-bytes { + type yang:zero-based-counter64; + description + "Internally generated ICMPv4 error packets dropped because of current + policy."; + } + + leaf drop-no-dest-softwire-ipv4-bytes { + type yang:zero-based-counter64; + description + "No matching destination softwire in the binding table; incremented + whether or not the reason was RFC7596."; + } + + leaf drop-in-by-rfc7596-icmpv4-bytes { + type yang:zero-based-counter64; + description + "Incoming ICMPv4 packets with no destination (RFC 7596 section 8.1)."; + } + + /* Fragmentation counters IPv4 */ + leaf drop-ipv4-frag-disabled { + type yang:zero-based-counter64; + description + "If fragmentation is disabled, the only potentially non-zero IPv4 + fragmentation counter is drop-ipv4-frag-disabled. If fragmentation is + enabled, it will always be zero."; + } + + leaf in-ipv4-frag-needs-reassembly { + type yang:zero-based-counter64; + description "An IPv4 fragment was received."; + } + + leaf in-ipv4-frag-reassembled { + type yang:zero-based-counter64; + description "A packet was successfully reassembled from IPv4 fragments."; + } + + leaf in-ipv4-frag-reassembly-unneeded { + type yang:zero-based-counter64; + description + "An IPv4 packet which was not a fragment was received - consequently, + it did not need to be reassembled. This should be the usual case."; + } + + leaf drop-ipv4-frag-invalid-reassembly { + type yang:zero-based-counter64; + description + "Two or more IPv4 fragments were received, and reassembly was started, + but was invalid and dropped. Causes include multiple fragments claiming + they are the last fragment, overlapping fragment offsets, or the packet + was being reassembled from too many fragments (the setting is + max_fragments_per_reassembly_packet, and the default is that no packet + should be reassembled from more than 40)."; + } + + leaf drop-ipv4-frag-random-evicted { + type yang:zero-based-counter64; + description + "Reassembling an IPv4 packet from fragments was in progress, but the + configured amount of packets to reassemble at once was exceeded, so one + was dropped at random. Consider increasing the setting + max_ipv4_reassembly_packets."; + } + + leaf out-ipv4-frag { + type yang:zero-based-counter64; + description + "An outgoing packet exceeded the configured IPv4 MTU, so needed to be + fragmented. This may happen, but should be unusual."; + } + + leaf out-ipv4-frag-not { + type yang:zero-based-counter64; + description + "An outgoing packet was small enough to pass through unfragmented - this + should be the usual case."; + } + + leaf memuse-ipv4-frag-reassembly-buffer { + type yang:zero-based-counter64; + description + "The amount of memory being used by the statically sized data structure + for reassembling IPv4 fragments. This is directly proportional to the + setting max_ipv4_reassembly_packets."; + } + + /* Fragmentation counters IPv6 */ + leaf drop-ipv6-frag-disabled { + type yang:zero-based-counter64; + description + "If fragmentation is disabled, the only potentially non-zero IPv6 + fragmentation counter is drop-ipv6-frag-disabled. If fragmentation is + enabled, it will always be zero."; + } + + leaf in-ipv6-frag-needs-reassembly { + type yang:zero-based-counter64; + description "An IPv6 fragment was received."; + } + + leaf in-ipv6-frag-reassembled { + type yang:zero-based-counter64; + description "A packet was successfully reassembled from IPv6 fragments."; + } + + leaf in-ipv6-frag-reassembly-unneeded { + type yang:zero-based-counter64; + description + "An IPv6 packet which was not a fragment was received - consequently, it + did not need to be reassembled. This should be the usual case."; + } + + leaf drop-ipv6-frag-invalid-reassembly { + type yang:zero-based-counter64; + description + "Two or more IPv6 fragments were received, and reassembly was started, + but was invalid and dropped. Causes include multiple fragments claiming + they are the last fragment, overlapping fragment offsets, or the packet + was being reassembled from too many fragments (the setting is + max_fragments_per_reassembly_packet, and the default is that no packet + should be reassembled from more than 40)."; + } + + leaf drop-ipv6-frag-random-evicted { + type yang:zero-based-counter64; + description + "Reassembling an IPv6 packet from fragments was in progress, but the + configured amount of packets to reassemble at once was exceeded, so one + was dropped at random. Consider increasing the setting + max_ipv6_reassembly_packets."; + } + + leaf out-ipv6-frag { + type yang:zero-based-counter64; + description + "An outgoing packet exceeded the configured IPv6 MTU, so needed to be + fragmented. This may happen, but should be unusual."; + } + + leaf out-ipv6-frag-not { + type yang:zero-based-counter64; + description + "An outgoing packet was small enough to pass through unfragmented - this + should be the usual case."; + } + + leaf memuse-ipv6-frag-reassembly-buffer { + type yang:zero-based-counter64; + description + "The amount of memory being used by the statically sized data structure + for reassembling IPv6 fragments. This is directly proportional to the + setting max_ipv6_reassembly_packets."; + } + } +} \ No newline at end of file diff --git a/src/program/config/get_state/README.inc b/src/program/config/get_state/README.inc new file mode 100644 index 0000000000..f72d4a0ce5 --- /dev/null +++ b/src/program/config/get_state/README.inc @@ -0,0 +1,9 @@ +Data Format Usage: + snabb config get-state + +This command takes in a instance name and a path to inside the schema and will +display the statistics at that path. + +Example usage: + + $ snabb config get-state lwaftr1 /snabb-softwire-v1/softwire-state/ diff --git a/src/program/config/get_state/get_state.lua b/src/program/config/get_state/get_state.lua new file mode 100644 index 0000000000..72cb7c4459 --- /dev/null +++ b/src/program/config/get_state/get_state.lua @@ -0,0 +1,134 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local lib = require("core.lib") +local shm = require("core.shm") +local xpath = require("lib.yang.xpath") +local yang = require("lib.yang.yang") +local yang_data = require("lib.yang.data") +local counter = require("core.counter") + +local counter_directory = "/apps" + +local function find_counters(pid) + local path = shm.root.."/"..pid..counter_directory + local counters = {} + for _, c in pairs(lib.files_in_directory(path)) do + local counterdir = "/"..pid..counter_directory.."/"..c + counters[c] = shm.open_frame(counterdir) + end + return counters +end + +local function flatten(t) + local rtn = {} + for k, v in pairs(t) do + if type(v) == "table" then + v = flatten(v) + for k1, v1 in pairs(v) do rtn[k1] = v1 end + else + rtn[k] = v + end + end + return rtn +end + +function collect_state_leaves(schema) + -- Iterate over schema looking fo state leaves at a specific path into the + -- schema. This should return a dictionary of leaf to lua path. + local function collection(scm, path, config) + local function newpath(oldpath) + return lib.deepcopy(oldpath) + end + if path == nil then path = {} end + + -- Add the current schema node to the path + table.insert(path, scm.id) + + if scm.config ~= nil then + config = scm.config + end + + if scm.kind == "container" then + -- Iterate over the body and recursively call self on all children. + local rtn = {} + for _, child in pairs(scm.body) do + local leaves = collection(child, newpath(path), config) + table.insert(rtn, leaves) + end + return rtn + elseif scm.kind == "leaf" then + if config == false then + local rtn = {} + rtn[path] = scm.id + return rtn + end + elseif scm.kind == "module" then + local rtn = {} + for _, v in pairs(scm.body) do + -- We deliberately don't want to include the module in the path. + table.insert(rtn, collection(v, {}, config)) + end + return rtn + end + end + + return flatten(collection(schema)) +end + +local function set_data_value(data, path, value) + local head = yang_data.normalize_id(table.remove(path, 1)) + if #path == 0 then + data[head] = value + return + end + if data[head] == nil then data[head] = {} end + set_data_value(data[head], path, value) +end + +local function show_state(counters, scm, path) + local leaves = collect_state_leaves(scm) + local data = {} + for leaf_path, leaf in pairs(leaves) do + for _, counter in pairs(counters) do + if counter[leaf] then + set_data_value(data, leaf_path, counter[leaf]) + end + end + end + yang_data.data_printer_from_schema(scm)(data, io.stdout) +end + +local function show_usage(status) + print(require("program.config.get_state.README_inc")) + main.exit(status) +end + +local function parse_args(args) + local handlers = {} + handlers.h = function() show_usage(0) end + args = lib.dogetopt(args, handlers, "h", {help="h"}) + if #args ~= 2 then show_usage(1) end + return unpack(args) +end + +function run(args) + local name, raw_path = parse_args(args) + local mod, path = xpath.load_from_path(raw_path) + + -- Find the PID of the name. + local pid = engine.enumerate_named_programs()[name] + + if pid == nil then + error("No app found with the name '"..name.."'.") + end + + local counters = find_counters(pid) + local s = yang.load_schema_by_name(mod) + + show_state(counters, s, path) +end + +function selftest() + +end \ No newline at end of file From d6340aebe1d869ef45a891479ecc9e26ae331fcf Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 18 Nov 2016 15:05:23 +0100 Subject: [PATCH 311/631] Add snabb config get-state as RPC This refactors a lot of the snabb config get-state code into lib.yang.state. This then adds an RPC call both in the get-state config program and the leader. --- src/apps/config/leader.lua | 5 + src/core/lib.lua | 14 +++ src/lib/yang/snabb-config-leader-v1.yang | 8 ++ src/lib/yang/util.lua | 5 + src/program/config/get_state/get_state.lua | 136 ++------------------- 5 files changed, 40 insertions(+), 128 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index df0efcb6b1..0460a42fd5 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -10,6 +10,7 @@ local yang = require("lib.yang.yang") local data = require("lib.yang.data") local util = require("lib.yang.util") local rpc = require("lib.yang.rpc") +local state = require("lib.yang.state") local path_mod = require("lib.yang.path") local app = require("core.app") local shm = require("core.shm") @@ -428,6 +429,10 @@ function Leader:rpc_attach_listener (args) return {} end +function Leader:rpc_get_state (args) + return {state=state.show_state(S.getpid(), args.path)} +end + function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end diff --git a/src/core/lib.lua b/src/core/lib.lua index 5f56904b4d..4df16a5be3 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -740,6 +740,20 @@ function parse (arg, config) return ret end +-- Flattens a multi-dimentional table +function flatten (val) + local rtn = {} + for k, v in pairs(val) do + if type(v) == "table" then + v = flatten(v) + for k1, v1 in pairs(v) do rtn[k1] = v1 end + else + rtn[k] = v + end + end + return rtn +end + function selftest () print("selftest: lib") print("Testing equal") diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index 9c0fb03e51..bfa1078f42 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -53,6 +53,14 @@ module snabb-config-leader-v1 { leaf schema { type string; mandatory true; } leaf revision { type string; } leaf path { type string; mandatory true; } + rpc get-state { + input { + leaf schema { type string; mandatory true; } + leaf revision { type string; } + leaf path { type string; default "/"; } + } + output { + leaf state { type string; } } } diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 339f5247d7..7c95b2d4b7 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -105,5 +105,10 @@ function selftest() assert(tointeger('-0x8000000000000000') == -0x8000000000000000LL) assert(ipv4_pton('255.0.0.1') == 255 * 2^24 + 1) assert(ipv4_ntop(ipv4_pton('255.0.0.1')) == '255.0.0.1') + + local ffile = FakeFile.new() + ffile:write("This is a test!") + ffile:write("\nI am not a real file") + assert(ffile.contents == "This is a test!\nI am not a real file") print('selftest: ok') end diff --git a/src/program/config/get_state/get_state.lua b/src/program/config/get_state/get_state.lua index 72cb7c4459..b16eded846 100644 --- a/src/program/config/get_state/get_state.lua +++ b/src/program/config/get_state/get_state.lua @@ -1,134 +1,14 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -local lib = require("core.lib") -local shm = require("core.shm") -local xpath = require("lib.yang.xpath") -local yang = require("lib.yang.yang") -local yang_data = require("lib.yang.data") -local counter = require("core.counter") - -local counter_directory = "/apps" - -local function find_counters(pid) - local path = shm.root.."/"..pid..counter_directory - local counters = {} - for _, c in pairs(lib.files_in_directory(path)) do - local counterdir = "/"..pid..counter_directory.."/"..c - counters[c] = shm.open_frame(counterdir) - end - return counters -end - -local function flatten(t) - local rtn = {} - for k, v in pairs(t) do - if type(v) == "table" then - v = flatten(v) - for k1, v1 in pairs(v) do rtn[k1] = v1 end - else - rtn[k] = v - end - end - return rtn -end - -function collect_state_leaves(schema) - -- Iterate over schema looking fo state leaves at a specific path into the - -- schema. This should return a dictionary of leaf to lua path. - local function collection(scm, path, config) - local function newpath(oldpath) - return lib.deepcopy(oldpath) - end - if path == nil then path = {} end - - -- Add the current schema node to the path - table.insert(path, scm.id) - - if scm.config ~= nil then - config = scm.config - end - - if scm.kind == "container" then - -- Iterate over the body and recursively call self on all children. - local rtn = {} - for _, child in pairs(scm.body) do - local leaves = collection(child, newpath(path), config) - table.insert(rtn, leaves) - end - return rtn - elseif scm.kind == "leaf" then - if config == false then - local rtn = {} - rtn[path] = scm.id - return rtn - end - elseif scm.kind == "module" then - local rtn = {} - for _, v in pairs(scm.body) do - -- We deliberately don't want to include the module in the path. - table.insert(rtn, collection(v, {}, config)) - end - return rtn - end - end - - return flatten(collection(schema)) -end - -local function set_data_value(data, path, value) - local head = yang_data.normalize_id(table.remove(path, 1)) - if #path == 0 then - data[head] = value - return - end - if data[head] == nil then data[head] = {} end - set_data_value(data[head], path, value) -end - -local function show_state(counters, scm, path) - local leaves = collect_state_leaves(scm) - local data = {} - for leaf_path, leaf in pairs(leaves) do - for _, counter in pairs(counters) do - if counter[leaf] then - set_data_value(data, leaf_path, counter[leaf]) - end - end - end - yang_data.data_printer_from_schema(scm)(data, io.stdout) -end - -local function show_usage(status) - print(require("program.config.get_state.README_inc")) - main.exit(status) -end - -local function parse_args(args) - local handlers = {} - handlers.h = function() show_usage(0) end - args = lib.dogetopt(args, handlers, "h", {help="h"}) - if #args ~= 2 then show_usage(1) end - return unpack(args) -end +local common = require("program.config.common") function run(args) - local name, raw_path = parse_args(args) - local mod, path = xpath.load_from_path(raw_path) - - -- Find the PID of the name. - local pid = engine.enumerate_named_programs()[name] - - if pid == nil then - error("No app found with the name '"..name.."'.") - end - - local counters = find_counters(pid) - local s = yang.load_schema_by_name(mod) - - show_state(counters, s, path) -end - -function selftest() - + args = common.parse_command_line(args, {command='get-state', with_path=true}) + local response = common.call_leader( + args.instance_id, 'get-state', + { schema = args.schema_name, revision = args.revision_date, + path = args.path }) + print(response.state) + main.exit(0) end \ No newline at end of file From 861989babe32c65cd85065559f52eec09d6d10b2 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 18 Nov 2016 15:36:08 +0100 Subject: [PATCH 312/631] Add missing lib.yang.state - oops! --- src/lib/yang/state.lua | 92 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 src/lib/yang/state.lua diff --git a/src/lib/yang/state.lua b/src/lib/yang/state.lua new file mode 100644 index 0000000000..ba37c7e2e6 --- /dev/null +++ b/src/lib/yang/state.lua @@ -0,0 +1,92 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local lib = require("core.lib") +local shm = require("core.shm") +local xpath = require("lib.yang.xpath") +local yang = require("lib.yang.yang") +local yang_data = require("lib.yang.data") +local counter = require("core.counter") + +local counter_directory = "/apps" + +local function find_counters(pid) + local path = shm.root.."/"..pid..counter_directory + local counters = {} + for _, c in pairs(lib.files_in_directory(path)) do + local counterdir = "/"..pid..counter_directory.."/"..c + counters[c] = shm.open_frame(counterdir) + end + return counters +end + +function collect_state_leaves(schema) + -- Iterate over schema looking fo state leaves at a specific path into the + -- schema. This should return a dictionary of leaf to lua path. + local function collection(scm, path, config) + local function newpath(oldpath) + return lib.deepcopy(oldpath) + end + if path == nil then path = {} end + + -- Add the current schema node to the path + table.insert(path, scm.id) + + if scm.config ~= nil then + config = scm.config + end + + if scm.kind == "container" then + -- Iterate over the body and recursively call self on all children. + local rtn = {} + for _, child in pairs(scm.body) do + local leaves = collection(child, newpath(path), config) + table.insert(rtn, leaves) + end + return rtn + elseif scm.kind == "leaf" then + if config == false then + local rtn = {} + rtn[path] = scm.id + return rtn + end + elseif scm.kind == "module" then + local rtn = {} + for _, v in pairs(scm.body) do + -- We deliberately don't want to include the module in the path. + table.insert(rtn, collection(v, {}, config)) + end + return rtn + end + end + + return lib.flatten(collection(schema)) +end + +local function set_data_value(data, path, value) + local head = yang_data.normalize_id(table.remove(path, 1)) + if #path == 0 then + data[head] = value + return + end + if data[head] == nil then data[head] = {} end + set_data_value(data[head], path, value) +end + +local function show_state(pid, path) + local counters = find_counters(pid) + local mod, path = xpath.load_from_path(raw_path) + local scm = yang.load_schema_by_name(mod) + local leaves = collect_state_leaves(scm) + local data = {} + for leaf_path, leaf in pairs(leaves) do + for _, counter in pairs(counters) do + if counter[leaf] then + set_data_value(data, leaf_path, counter[leaf]) + end + end + end + local fakeout = util.FakeFile.new() + yang_data.data_printer_from_schema(scm)(data, fakeout) + return fakeout.contents +end \ No newline at end of file From f27638e5e37ea28c0428cacd8ddc9beb5290132a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 24 Nov 2016 12:15:58 +0100 Subject: [PATCH 313/631] Add verious fixes for snabb config get-state --- src/apps/config/leader.lua | 5 +++- src/lib/yang/path.lua | 2 ++ src/lib/yang/schema.lua | 12 +++++++- src/lib/yang/snabb-config-leader-v1.yang | 3 ++ src/lib/yang/state.lua | 37 ++++++++++++------------ src/lib/yang/util.lua | 5 ---- src/program/config/get_state/README.inc | 10 ++++--- 7 files changed, 44 insertions(+), 30 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 0460a42fd5..fb45251616 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -430,7 +430,10 @@ function Leader:rpc_attach_listener (args) end function Leader:rpc_get_state (args) - return {state=state.show_state(S.getpid(), args.path)} + assert(args.schema == self.schema_name) + local printer = path_printer_for_schema_by_name(self.schema_name, args.path) + local state = state.show_state(self.schema_name, S.getpid(), args.path) + return {state=printer(state, yang.string_output_file())} end function Leader:handle (payload) diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 466ea0737e..9a5a541b32 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -82,6 +82,8 @@ function convert_path(grammar, path) return assert(handlers[grammar.type], grammar.type)(grammar, fragment) end + if path == "/" then return {} end + local ret = {} local node = grammar if path:sub(1, 1) == "/" then path = path:sub(2) end -- remove leading / diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 9018b178b6..70fef553bc 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -571,6 +571,7 @@ local primitive_types = set( -- Resolve if-feature. -- Warn on any "when", resolving them as being true. -- Resolve all augment and refine nodes. (TODO) +-- Inherits config attributes from parents function resolve(schema, features) local function shallow_copy(node) local out = {} @@ -682,6 +683,12 @@ function resolve(schema, features) return nil, env end end + -- Inherit config attribute from parent unless we have it set. + if node.config == nil then + node.config = env.env.config + end + env.config = node.config + if node.type then node.type = visit_type(node.type, env) if not node.primitive_type then @@ -700,7 +707,10 @@ function resolve(schema, features) node.body[k] = v end -- TODO: Handle refine and augment statements. - else + end + end + for k,v in pairs(node.body or {}) do + if v.kind ~= "uses" then node.body[k] = visit(v, env) end end diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index bfa1078f42..4f13a7d455 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -53,6 +53,9 @@ module snabb-config-leader-v1 { leaf schema { type string; mandatory true; } leaf revision { type string; } leaf path { type string; mandatory true; } + } + } + rpc get-state { input { leaf schema { type string; mandatory true; } diff --git a/src/lib/yang/state.lua b/src/lib/yang/state.lua index ba37c7e2e6..a4a35417d7 100644 --- a/src/lib/yang/state.lua +++ b/src/lib/yang/state.lua @@ -3,7 +3,7 @@ module(..., package.seeall) local lib = require("core.lib") local shm = require("core.shm") -local xpath = require("lib.yang.xpath") +local xpath = require("lib.yang.path") local yang = require("lib.yang.yang") local yang_data = require("lib.yang.data") local counter = require("core.counter") @@ -23,29 +23,23 @@ end function collect_state_leaves(schema) -- Iterate over schema looking fo state leaves at a specific path into the -- schema. This should return a dictionary of leaf to lua path. - local function collection(scm, path, config) + local function collection(scm, path) local function newpath(oldpath) return lib.deepcopy(oldpath) end if path == nil then path = {} end - - -- Add the current schema node to the path table.insert(path, scm.id) - if scm.config ~= nil then - config = scm.config - end - if scm.kind == "container" then -- Iterate over the body and recursively call self on all children. local rtn = {} for _, child in pairs(scm.body) do - local leaves = collection(child, newpath(path), config) + local leaves = collection(child, newpath(path)) table.insert(rtn, leaves) end return rtn elseif scm.kind == "leaf" then - if config == false then + if scm.config == false then local rtn = {} rtn[path] = scm.id return rtn @@ -54,13 +48,17 @@ function collect_state_leaves(schema) local rtn = {} for _, v in pairs(scm.body) do -- We deliberately don't want to include the module in the path. - table.insert(rtn, collection(v, {}, config)) + table.insert(rtn, collection(v, {})) end return rtn end + return {} end - return lib.flatten(collection(schema)) + local leaves = collection(schema) + if leaves == nil then return {} end + leaves = lib.flatten(leaves) + return function () return leaves end end local function set_data_value(data, path, value) @@ -73,11 +71,14 @@ local function set_data_value(data, path, value) set_data_value(data[head], path, value) end -local function show_state(pid, path) +function show_state(scm, pid, raw_path) + local schema = yang.load_schema_by_name(scm) + local grammar = yang_data.data_grammar_from_schema(schema) local counters = find_counters(pid) - local mod, path = xpath.load_from_path(raw_path) - local scm = yang.load_schema_by_name(mod) - local leaves = collect_state_leaves(scm) + local path = xpath.convert_path(grammar, raw_path) + + -- Lookup the specific schema element that's being addressed by the path + local leaves = collect_state_leaves(schema)() local data = {} for leaf_path, leaf in pairs(leaves) do for _, counter in pairs(counters) do @@ -86,7 +87,5 @@ local function show_state(pid, path) end end end - local fakeout = util.FakeFile.new() - yang_data.data_printer_from_schema(scm)(data, fakeout) - return fakeout.contents + return data end \ No newline at end of file diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 7c95b2d4b7..339f5247d7 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -105,10 +105,5 @@ function selftest() assert(tointeger('-0x8000000000000000') == -0x8000000000000000LL) assert(ipv4_pton('255.0.0.1') == 255 * 2^24 + 1) assert(ipv4_ntop(ipv4_pton('255.0.0.1')) == '255.0.0.1') - - local ffile = FakeFile.new() - ffile:write("This is a test!") - ffile:write("\nI am not a real file") - assert(ffile.contents == "This is a test!\nI am not a real file") print('selftest: ok') end diff --git a/src/program/config/get_state/README.inc b/src/program/config/get_state/README.inc index f72d4a0ce5..ac151e9246 100644 --- a/src/program/config/get_state/README.inc +++ b/src/program/config/get_state/README.inc @@ -1,9 +1,11 @@ Data Format Usage: - snabb config get-state + snabb config get-state -This command takes in a instance name and a path to inside the schema and will -display the statistics at that path. +This command takes in a instance identifier (a name or a PID) and a path to +inside the schema and will display the statistics at that path. The program +will find the conters specified under the path and match those to counters +defined in the apps. Example usage: - $ snabb config get-state lwaftr1 /snabb-softwire-v1/softwire-state/ + $ snabb config get-state lwaftr1 /softwire-state/ From 14542942bd8dca02e2a156831507a2fa0cdfd4b9 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 25 Nov 2016 15:41:21 +0000 Subject: [PATCH 314/631] Fix some nits and add selftest --- src/core/lib.lua | 14 ------ src/lib/yang/path.lua | 1 + src/lib/yang/schema.lua | 4 +- src/lib/yang/state.lua | 104 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 105 insertions(+), 18 deletions(-) diff --git a/src/core/lib.lua b/src/core/lib.lua index 4df16a5be3..5f56904b4d 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -740,20 +740,6 @@ function parse (arg, config) return ret end --- Flattens a multi-dimentional table -function flatten (val) - local rtn = {} - for k, v in pairs(val) do - if type(v) == "table" then - v = flatten(v) - for k1, v1 in pairs(v) do rtn[k1] = v1 end - else - rtn[k] = v - end - end - return rtn -end - function selftest () print("selftest: lib") print("Testing equal") diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 9a5a541b32..95d08e9d5b 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -64,6 +64,7 @@ end -- Converts an XPath path to a lua array consisting of path componants. -- A path component can then be resolved on a yang data tree: function convert_path(grammar, path) + local path = normalize_path(path) local handlers = {} function handlers.scalar(grammar, fragment) return {name=fragment.name, grammar=grammar} diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 70fef553bc..a98529e645 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -710,9 +710,7 @@ function resolve(schema, features) end end for k,v in pairs(node.body or {}) do - if v.kind ~= "uses" then - node.body[k] = visit(v, env) - end + node.body[k] = visit(v, env) end end return node, env diff --git a/src/lib/yang/state.lua b/src/lib/yang/state.lua index a4a35417d7..0f2f93a579 100644 --- a/src/lib/yang/state.lua +++ b/src/lib/yang/state.lua @@ -10,6 +10,19 @@ local counter = require("core.counter") local counter_directory = "/apps" +local function flatten(val) + local rtn = {} + for k, v in pairs(val) do + if type(v) == "table" then + v = flatten(v) + for k1, v1 in pairs(v) do rtn[k1] = v1 end + else + rtn[k] = v + end + end + return rtn +end + local function find_counters(pid) local path = shm.root.."/"..pid..counter_directory local counters = {} @@ -57,7 +70,7 @@ function collect_state_leaves(schema) local leaves = collection(schema) if leaves == nil then return {} end - leaves = lib.flatten(leaves) + leaves = flatten(leaves) return function () return leaves end end @@ -88,4 +101,93 @@ function show_state(scm, pid, raw_path) end end return data +end + +function selftest () + print("selftest: lib.yang.state") + local simple_router_schema_src = [[module snabb-simple-router { + namespace snabb:simple-router; + prefix simple-router; + + import ietf-inet-types {prefix inet;} + + leaf active { type boolean; default true; } + leaf-list blocked-ips { type inet:ipv4-address; } + + container routes { + presence true; + list route { + key addr; + leaf addr { type inet:ipv4-address; mandatory true; } + leaf port { type uint8 { range 0..11; } mandatory true; } + } + + } + + + + container state { + presence true; + config false; + + leaf total-packets { + type uint64 { + default 0; + } + } + + leaf dropped-packets { + type uint64 { + default 0; + } + } + } + + grouping detailed-counters { + leaf dropped-wrong-route { + type uint64 { default 0; } + } + leaf dropped-not-permitted { + type uint64 { default 0; } + } + } + + container detailed-state { + presence true; + config false; + uses "detailed-counters"; + } + }]] + local function table_length(tbl) + local rtn = 0 + for k,v in pairs(tbl) do rtn = rtn + 1 end + return rtn + end + local function in_array(needle, haystack) + for _, i in pairs(haystack) do if needle == i then return true end end + return false + end + + local simple_router_schema = yang.load_schema(simple_router_schema_src, + "state-test") + local leaves = collect_state_leaves(simple_router_schema)() + + -- Check the correct number of leaves have been found + assert(table_length(leaves) == 4) + + -- Check it's found every state path. + local state_leaves = { + "total-packets", + "dropped-packets", + "dropped-wrong-route", + "dropped-not-permitted" + } + for _, leaf in pairs(leaves) do + assert(in_array(leaf, state_leaves)) + end + + -- Check flatten produces a single dimentional table with all the elements. + local multi_dimentional = {{hello="hello"}, {world="world"}} + assert(flatten(multi_dimentional), {hello="hello", world="world"}) + print("selftest: ok") end \ No newline at end of file From ff6c5e4557b3664836c9bcc27ba77d255785efa9 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 25 Nov 2016 17:36:21 +0100 Subject: [PATCH 315/631] Add assert in core.lib.files_in_directory for better errors --- src/core/lib.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib.lua b/src/core/lib.lua index 5f56904b4d..4fd45a06e4 100644 --- a/src/core/lib.lua +++ b/src/core/lib.lua @@ -109,7 +109,7 @@ function firstline (filename) return readfile(filename, "*l") end function files_in_directory (dir) local files = {} - for line in io.popen('ls -1 "'..dir..'" 2>/dev/null'):lines() do + for line in assert(io.popen('ls -1 "'..dir..'" 2>/dev/null')):lines() do table.insert(files, line) end return files From e479e6188f52da8ab1463e5c306f528356def99f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 25 Nov 2016 18:02:18 +0100 Subject: [PATCH 316/631] Add "snabb ps" --- src/program/ps/README | 8 ++++++++ src/program/ps/README.inc | 1 + src/program/ps/ps.lua | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 src/program/ps/README create mode 120000 src/program/ps/README.inc create mode 100644 src/program/ps/ps.lua diff --git a/src/program/ps/README b/src/program/ps/README new file mode 100644 index 0000000000..16eebb2d0c --- /dev/null +++ b/src/program/ps/README @@ -0,0 +1,8 @@ +Usage: snabb ps [OPTION]... +Print a list of running Snabb instances. + +Available options: + -h, --help Display this message. + +The default output format is one PID per line. More options can be +added in the future. diff --git a/src/program/ps/README.inc b/src/program/ps/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/ps/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua new file mode 100644 index 0000000000..00874a9291 --- /dev/null +++ b/src/program/ps/ps.lua @@ -0,0 +1,38 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(..., package.seeall) + +local S = require("syscall") +local lib = require("core.lib") +local shm = require("core.shm") + +local function usage (code) + local f = code == 0 and io.stdout or io.stderr + f:write(require("program.top.README_inc")) + main.exit(code) +end + +local function parse_args (args) + local opt = {} + function opt.h (arg) usage(0) end + args = lib.dogetopt(args, opt, "h", {help='h'}) + if #args ~= 0 then usage(1) end +end + +local function compute_snabb_instances() + -- Produces set of snabb instances, excluding this one. + local pids = {} + local my_pid = S.getpid() + for _, name in ipairs(shm.children("/")) do + -- This could fail as the name could be for example "by-name" + local p = tonumber(name) + if p and p ~= my_pid then table.insert(pids, p) end + end + return pids +end + +function run (args) + local instances = compute_snabb_instances() + for _, instance in ipairs(instances) do print(instance) end + main.exit(0) +end From 7adc70840dfd9eb06b685ab2a291dc2550c47dd3 Mon Sep 17 00:00:00 2001 From: Luke Gorrie Date: Sat, 26 Nov 2016 09:22:01 +0000 Subject: [PATCH 317/631] ljsyscall: Add more protection against fd double-close This is a downstream cherry-pick of this change: https://github.com/justincormack/ljsyscall/pull/205 Required to prevent a double-close problem in memory.lua. --- lib/ljsyscall/syscall/syscalls.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/ljsyscall/syscall/syscalls.lua b/lib/ljsyscall/syscall/syscalls.lua index 8ff4c5358c..e76fe725d1 100644 --- a/lib/ljsyscall/syscall/syscalls.lua +++ b/lib/ljsyscall/syscall/syscalls.lua @@ -75,7 +75,13 @@ local function retiter(ret, err, array) end -- generic system calls -function S.close(fd) return retbool(C.close(getfd(fd))) end +function S.close(fd) + if fd == getfd(fd) then -- fd number + return retbool(C.close(getfd(fd))) + else -- fd object: avoid mulitple close + return fd:close() + end +end function S.chdir(path) return retbool(C.chdir(path)) end function S.fchdir(fd) return retbool(C.fchdir(getfd(fd))) end function S.fchmod(fd, mode) return retbool(C.fchmod(getfd(fd), c.MODE[mode])) end From aacb39488835426e934d13e7848596dad8d8ac45 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 10:46:30 +0100 Subject: [PATCH 318/631] Add 3.0.1 release notes --- src/program/lwaftr/doc/CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index f965c1ff34..cad7f2186d 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,17 @@ # Change Log +## [3.0.1] - 2016-11-28 + +A release to finish "snabb config" features. + + * New "snabb config" commands "add", "remove", and "listen". See [the + `snabb config` documentation](../../config/README.md) for full + details. + + * The "get", "set", "add", and "remove" "snabb config" commands can now + take paths to indicate sub-configurations on which to operate. This + was documented before but not yet implemented. + ## [3.0.0] - 2016-11-18 A change to migrate the lwAFTR to use a new YANG-based configuration. From ebd2a4b22ed37c208b9c8861e9aef2e2ba843eaf Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 25 Nov 2016 14:09:17 +0100 Subject: [PATCH 319/631] Fix bug in core.app where shm.mkdir behavour had changed --- src/core/app.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 2c1c44b6c5..c738bd50af 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -138,17 +138,17 @@ end -- an error. function claim_name(name) configuration[name] = name - local namedir = "by-name/" .. name + local bynameidr = "by-name/" local namedir_fq = named_program_root .. "/" .. name local piddir = shm.root .. "/" .. S.getpid() local backlinkdir = piddir.."/name" -- Verify that the by-name directory exists. - shm.mkdir(namedir) + shm.mkdir(bynameidr) -- Verify that we've not already claimed a name assert(configuration.name == nil, "Name already claimed, cannot claim: "..name) - + -- Create the new symlink. assert(S.symlink(piddir, namedir_fq)) From 699150779fcde1316bc6e10d2883bb7287e7f1bd Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 28 Nov 2016 11:09:40 +0100 Subject: [PATCH 320/631] Added lib.dirname to channel's mkdir --- src/apps/config/channel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/config/channel.lua b/src/apps/config/channel.lua index c13dcbd103..49c94c2186 100644 --- a/src/apps/config/channel.lua +++ b/src/apps/config/channel.lua @@ -27,7 +27,7 @@ local ring_buffer_t = ffi.typeof([[struct { -- A: We need a variable-sized mapping. local function create_ring_buffer (name, size) local path = shm.resolve(name) - shm.mkdir(path) + shm.mkdir(lib.dirname(path)) path = shm.root..'/'..path local fd, err = S.open(path, "creat, rdwr, excl", '0664') if not fd then From 4e8a94f3311f4cbd6955cd3c6ef46be96c2834e8 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 25 Nov 2016 14:25:09 +0100 Subject: [PATCH 321/631] Fix typo, idr->dir --- src/core/app.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index c738bd50af..122fa312b3 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -138,13 +138,13 @@ end -- an error. function claim_name(name) configuration[name] = name - local bynameidr = "by-name/" + local bynamedir = "by-name/" local namedir_fq = named_program_root .. "/" .. name local piddir = shm.root .. "/" .. S.getpid() local backlinkdir = piddir.."/name" -- Verify that the by-name directory exists. - shm.mkdir(bynameidr) + shm.mkdir(bynamedir) -- Verify that we've not already claimed a name assert(configuration.name == nil, "Name already claimed, cannot claim: "..name) From 33677ff127d8399dd6596acb6aac5bf48f1e966d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 10:58:56 +0100 Subject: [PATCH 322/631] Leader can load schema-specific support module Also add snabb-softwire-v1 example module. --- src/apps/config/leader.lua | 31 ++++++++++--------- src/apps/config/support/snabb-softwire-v1.lua | 11 +++++++ 2 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 src/apps/config/support/snabb-softwire-v1.lua diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index df0efcb6b1..38557efb3c 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -39,6 +39,19 @@ local function open_socket (file) return socket end +local generic_schema_config_support = { + compute_config_actions = function(old_graph, new_graph, verb, path, ...) + return app.compute_config_actions(old_graph, new_graph) + end +} + +local function load_schema_config_support(schema_name) + local mod_name = 'apps.config.support.'..schema_name:gsub('-', '_') + local success, support_mod = pcall(require, mod_name) + if success then return support_mod.get_config_support() end + return generic_schema_config_support +end + function Leader:new (conf) local ret = setmetatable({}, {__index=Leader}) ret.socket_file_name = conf.socket_file_name @@ -47,6 +60,7 @@ function Leader:new (conf) ret.socket_file_name = instance_dir..'/'..ret.socket_file_name end ret.schema_name = conf.schema_name + ret.support = load_schema_config_support(conf.schema_name) ret.socket = open_socket(ret.socket_file_name) ret.peers = {} ret.setup_fn = conf.setup_fn @@ -67,8 +81,8 @@ end function Leader:reset_configuration (configuration) local new_app_graph = self.setup_fn(configuration) - local actions = app.compute_config_actions(self.current_app_graph, - new_app_graph) + local actions = self.support.compute_config_actions( + self.current_app_graph, new_app_graph, 'load') self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph self.current_configuration = configuration @@ -120,16 +134,6 @@ function Leader:rpc_get_config (args) return { config = config } end -local generic_schema_support = { - compute_config_actions = function(old_graph, new_graph, verb, path, ...) - return app.compute_config_actions(old_graph, new_graph) - end -} - -local function load_schema_support(schema_name) - return generic_schema_support -end - local function path_parser_for_grammar(grammar, path) local getter, subgrammar = path_mod.resolver(grammar, path) return data.data_parser_from_grammar(subgrammar) @@ -383,8 +387,7 @@ function Leader:update_configuration (schema_name, update_fn, verb, path, ...) assert(schema_name == self.schema_name) local new_config = update_fn(self.current_configuration, ...) local new_app_graph = self.setup_fn(new_config) - local support = load_schema_support(schema_name) - local actions = support.compute_config_actions( + local actions = self.support.compute_config_actions( self.current_app_graph, new_app_graph, verb, path, ...) self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua new file mode 100644 index 0000000000..e5fb6c1a84 --- /dev/null +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -0,0 +1,11 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) +local app = require('core.app') + +local function compute_config_actions(old_graph, new_graph, verb, path, arg) + return app.compute_config_actions(old_graph, new_graph) +end + +function get_config_support() + return { compute_config_actions = compute_config_actions } +end From 1239f6796c45c1186f5134176acbeeca9d43e6e9 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 28 Nov 2016 11:34:21 +0100 Subject: [PATCH 323/631] Refactor out config inheritance from lib.yang.schema.resolver --- src/lib/yang/schema.lua | 99 ++++++++++++++++++++++++++++++++++------- 1 file changed, 82 insertions(+), 17 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index a98529e645..300c187785 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -12,12 +12,18 @@ local function assert_with_loc(expr, loc, msg, ...) return expr end +local function shallow_copy(node) + local out = {} + for k,v in pairs(node) do out[k] = v end + return out +end + -- (kind -> (function(Node) -> value)) local initializers = {} local function declare_initializer(init, ...) for _, keyword in ipairs({...}) do initializers[keyword] = init end end - + local function parse_node(src) local ret = {} ret.kind = assert(src.keyword, 'missing keyword') @@ -564,6 +570,26 @@ local primitive_types = set( 'binary', 'bits', 'boolean', 'decimal64', 'empty', 'enumeration', 'identityref', 'instance-identifier', 'leafref', 'string', 'union') +-- Inherits config attributes from parents +local function inherit_config(schema, env) + if env == nil then env = {env={}} end + + if schema.config ~= nil then + env.config = schema.config + elseif env.env.config ~= nil then + schema = shallow_copy(schema) + schema.config = env.env.config + end + + if schema.body then + for name, node in pairs(schema.body) do + schema.body[name] = inherit_config(node, {env=env}) + end + end + + return schema +end + -- Inline "grouping" into "uses". -- Inline "submodule" into "include". -- Inline "imports" into "module". @@ -571,13 +597,7 @@ local primitive_types = set( -- Resolve if-feature. -- Warn on any "when", resolving them as being true. -- Resolve all augment and refine nodes. (TODO) --- Inherits config attributes from parents function resolve(schema, features) - local function shallow_copy(node) - local out = {} - for k,v in pairs(node) do out[k] = v end - return out - end local function pop_prop(node, prop) local val = node[prop] node[prop] = nil @@ -683,11 +703,6 @@ function resolve(schema, features) return nil, env end end - -- Inherit config attribute from parent unless we have it set. - if node.config == nil then - node.config = env.env.config - end - env.config = node.config if node.type then node.type = visit_type(node.type, env) @@ -707,11 +722,10 @@ function resolve(schema, features) node.body[k] = v end -- TODO: Handle refine and augment statements. + else + node.body[k] = visit(v, env) end end - for k,v in pairs(node.body or {}) do - node.body[k] = visit(v, env) - end end return node, env end @@ -803,10 +817,12 @@ function parse_schema_file(filename) end function load_schema(src, filename) - return resolve(primitivize(parse_schema(src, filename))) + local s, e = resolve(primitivize(parse_schema(src, filename))) + return inherit_config(s), e end function load_schema_file(filename) - return resolve(primitivize(parse_schema_file(filename))) + local s, e = resolve(primitivize(parse_schema_file(filename))) + return inherit_config(s), e end function load_schema_by_name(name, revision) -- FIXME: @ is not valid in a Lua module name. @@ -931,5 +947,54 @@ function selftest() load_schema_by_name('ietf-yang-types') load_schema_by_name('ietf-softwire') load_schema_by_name('snabb-softwire-v1') + + local inherit_config_schema = [[module config-inheritance { + namespace cs; + prefix cs; + + container foo { + container bar { + config false; + + leaf baz { + type uint8; + } + + leaf qux { + config true; + type uint8; + } + } + } + + grouping quux { + leaf quuz { + type uint8; + } + } + + container corge { uses quux; } + container grault { config true; uses quux; } + container garply { config false; uses quux; } + }]] + + local icschema = load_schema(inherit_config_schema) + + -- Test things that should be null, still are. + assert(icschema.config == nil) + assert(icschema.body.foo.config == nil) + + -- Assert the regular config is propergated through container. + assert(icschema.body.foo.body.bar.config == false) + assert(icschema.body.foo.body.bar.body.baz.config == false) + assert(icschema.body.foo.body.bar.body.qux.config == true) + + -- Now test the grouping, we need to ensure copying is done correctly. + assert(icschema.body.corge.config == nil) + assert(icschema.body.corge.body.quuz.config == nil) + assert(icschema.body.grault.config == true) + assert(icschema.body.grault.body.quuz.config == true) + assert(icschema.body.garply.config == false) + assert(icschema.body.garply.body.quuz.config == false) print('selftest: ok') end From 8ad3595c2e4f0a8245f32973257dbbe48e9b9a5d Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 28 Nov 2016 11:54:54 +0100 Subject: [PATCH 324/631] Change parameter in inherit_config from table to boolean --- src/lib/yang/schema.lua | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 300c187785..bf4eeef301 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -571,19 +571,18 @@ local primitive_types = set( 'identityref', 'instance-identifier', 'leafref', 'string', 'union') -- Inherits config attributes from parents -local function inherit_config(schema, env) - if env == nil then env = {env={}} end +local function inherit_config(schema, config) if schema.config ~= nil then - env.config = schema.config - elseif env.env.config ~= nil then + config = schema.config + elseif config ~= nil then schema = shallow_copy(schema) - schema.config = env.env.config + schema.config = config end if schema.body then for name, node in pairs(schema.body) do - schema.body[name] = inherit_config(node, {env=env}) + schema.body[name] = inherit_config(node, config) end end From e2a0389236c87baaf11512535b22c95fbb75742b Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 28 Nov 2016 12:03:18 +0100 Subject: [PATCH 325/631] Add assert to ensure config can't be in state data --- src/lib/yang/schema.lua | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index bf4eeef301..677f6df2d7 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -572,8 +572,8 @@ local primitive_types = set( -- Inherits config attributes from parents local function inherit_config(schema, config) - if schema.config ~= nil then + assert(not config or schema.config == false) config = schema.config elseif config ~= nil then schema = shallow_copy(schema) @@ -958,11 +958,6 @@ function selftest() leaf baz { type uint8; } - - leaf qux { - config true; - type uint8; - } } } @@ -986,7 +981,6 @@ function selftest() -- Assert the regular config is propergated through container. assert(icschema.body.foo.body.bar.config == false) assert(icschema.body.foo.body.bar.body.baz.config == false) - assert(icschema.body.foo.body.bar.body.qux.config == true) -- Now test the grouping, we need to ensure copying is done correctly. assert(icschema.body.corge.config == nil) From c545c5f522e05f2b3d6e4572d87f4ccea30ec3e0 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 28 Nov 2016 12:10:25 +0100 Subject: [PATCH 326/631] Change use of shallow copy to make more robust --- src/lib/yang/schema.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 677f6df2d7..59515ee914 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -576,13 +576,12 @@ local function inherit_config(schema, config) assert(not config or schema.config == false) config = schema.config elseif config ~= nil then - schema = shallow_copy(schema) schema.config = config end if schema.body then for name, node in pairs(schema.body) do - schema.body[name] = inherit_config(node, config) + schema.body[name] = inherit_config(shallow_copy(node), config) end end From ccc176e45c80fa427592d90c91ae1a0a626d0510 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 28 Nov 2016 12:23:12 +0100 Subject: [PATCH 327/631] Second attempt to fix shallow_copy --- src/lib/yang/schema.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 59515ee914..99188d2ea6 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -576,12 +576,14 @@ local function inherit_config(schema, config) assert(not config or schema.config == false) config = schema.config elseif config ~= nil then + schema = shallow_copy(schema) schema.config = config end if schema.body then + schema.body = shallow_copy(schema.body) for name, node in pairs(schema.body) do - schema.body[name] = inherit_config(shallow_copy(node), config) + schema.body[name] = inherit_config(node, config) end end From 854cf1490ec7cbb7b90daf8b7ed390ddac09d36d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 12:45:34 +0100 Subject: [PATCH 328/631] Add lwAFTR v3.0.1 changelog entry --- src/program/lwaftr/doc/CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index cad7f2186d..8861f603b2 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -4,13 +4,13 @@ A release to finish "snabb config" features. - * New "snabb config" commands "add", "remove", and "listen". See [the - `snabb config` documentation](../../config/README.md) for full - details. + * New "snabb config" commands "get-stat", "add", "remove", and + "listen". See [the `snabb config` documentation](../../config/README.md) + for full details. - * The "get", "set", "add", and "remove" "snabb config" commands can now - take paths to indicate sub-configurations on which to operate. This - was documented before but not yet implemented. + * The "get-state", "get", "set", "add", and "remove" "snabb config" + commands can now take paths to indicate sub-configurations on which + to operate. This was documented before but not yet implemented. ## [3.0.0] - 2016-11-18 From 632120bae847c37091c492f4a85b042354411571 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 12:51:37 +0100 Subject: [PATCH 329/631] Fix typo --- src/program/lwaftr/doc/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 8861f603b2..ee98fa0c56 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -4,7 +4,7 @@ A release to finish "snabb config" features. - * New "snabb config" commands "get-stat", "add", "remove", and + * New "snabb config" commands "get-state", "add", "remove", and "listen". See [the `snabb config` documentation](../../config/README.md) for full details. From 2eb8aad4878da7fb423ea8d6d1f356bbd651f9fe Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 14:00:38 +0100 Subject: [PATCH 330/631] snabb ps parses command-line args --- src/program/ps/ps.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index 00874a9291..6a695eb6f6 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -8,7 +8,7 @@ local shm = require("core.shm") local function usage (code) local f = code == 0 and io.stdout or io.stderr - f:write(require("program.top.README_inc")) + f:write(require("program.ps.README_inc")) main.exit(code) end @@ -32,6 +32,7 @@ local function compute_snabb_instances() end function run (args) + parse_args (args) local instances = compute_snabb_instances() for _, instance in ipairs(instances) do print(instance) end main.exit(0) From ef9c5f04811a640f890530d134e04c6df17acf87 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 15:20:57 +0100 Subject: [PATCH 331/631] Fix bug in ctable:lookup_and_copy An embarrassing bug where I was somehow thinking too much in terms of C by-value structures. --- src/lib/ctable.lua | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index fdb94656de..39259c117c 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -300,7 +300,7 @@ end function CTable:lookup_and_copy(key, entry) local entry_ptr = self:lookup_ptr(key) if not entry_ptr then return false end - entry = entry_ptr + ffi.copy(entry, entry_ptr, ffi.sizeof(entry)) return true end @@ -603,6 +603,20 @@ function selftest() end ctab:selfcheck() + -- Incrementing by 31 instead of 1 just to save test time. + do + local entry = ctab.entry_type() + for i = 1, occupancy, 31 do + assert(ctab:lookup_and_copy(i, entry)) + assert(entry.key == i) + assert(entry.value[0] == bnot(i)) + ctab:remove(entry.key) + assert(ctab:lookup_ptr(i) == nil) + ctab:add(entry.key, entry.value) + assert(ctab:lookup_ptr(i).value[0] == bnot(i)) + end + end + local iterated = 0 for entry in ctab:iterate() do iterated = iterated + 1 end assert(iterated == occupancy) From e99622476c36f87d771eff8989376cc55e9a182a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 15:38:22 +0100 Subject: [PATCH 332/631] Add support for fast binding-table updates --- src/apps/config/action_codec.lua | 23 +++++++++- src/apps/config/follower.lua | 9 +++- src/apps/config/leader.lua | 2 +- src/apps/config/support/snabb-softwire-v1.lua | 46 ++++++++++++++++++- src/apps/lwaftr/binding_table.lua | 14 ++++++ src/apps/lwaftr/lwaftr.lua | 10 ++++ 6 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/apps/config/action_codec.lua b/src/apps/config/action_codec.lua index af8778e56a..6b53084166 100644 --- a/src/apps/config/action_codec.lua +++ b/src/apps/config/action_codec.lua @@ -11,7 +11,8 @@ local shm = require("core.shm") local action_names = { 'unlink_output', 'unlink_input', 'free_link', 'new_link', 'link_output', 'link_input', 'stop_app', - 'start_app', 'reconfig_app' } + 'start_app', 'reconfig_app', + 'call_app_method_with_blob' } local action_codes = {} for i, name in ipairs(action_names) do action_codes[name] = i end @@ -63,6 +64,12 @@ function actions.reconfig_app (codec, appname, class, arg) local config = codec:config(class, arg) return codec:finish(appname, _class, config) end +function actions.call_app_method_with_blob (codec, appname, methodname, blob) + local appname = codec:string(appname) + local methodname = codec:string(methodname) + local blob = codec:blob(blob) + return codec:finish(appname, methodname, blob) +end local public_names = {} local function find_public_name(obj) @@ -111,6 +118,10 @@ local function encoder() self:uint32(#str) table.insert(self.out, ffi.new('uint8_t[?]', #str, str)) end + function encoder:blob(blob) + self:uint32(ffi.sizeof(blob)) + table.insert(self.out, blob) + end function encoder:class(class) local require_path, name = find_public_name(class) self:string(require_path) @@ -164,6 +175,12 @@ local function decoder(buf, len) local len = self:uint32() return ffi.string(self:read(len), len) end + function decoder:blob() + local len = self:uint32() + local blob = ffi.new('uint8_t[?]', len) + ffi.copy(blob, self:read(len), len) + return blob + end function decoder:class() local require_path, name = self:string(), self:string() return assert(require(require_path)[name]) @@ -206,6 +223,9 @@ function selftest () end local appname, linkname, linkspec = 'foo', 'bar', 'foo.a -> bar.q' local class, arg = require('apps.basic.basic_apps').Tee, {} + -- Because lib.equal only returns true when comparing cdata of + -- exactly the same type, here we have to use uint8_t[?]. + local methodname, blob = 'zog', ffi.new('uint8_t[?]', 3, 1, 2, 3) test_action({'unlink_output', {appname, linkname}}) test_action({'unlink_input', {appname, linkname}}) test_action({'free_link', {linkspec}}) @@ -215,5 +235,6 @@ function selftest () test_action({'stop_app', {appname}}) test_action({'start_app', {appname, class, arg}}) test_action({'reconfig_app', {appname, class, arg}}) + test_action({'call_app_method_with_blob', {appname, methodname, blob}}) print('selftest: ok') end diff --git a/src/apps/config/follower.lua b/src/apps/config/follower.lua index 42eb3f52ad..83240f0836 100644 --- a/src/apps/config/follower.lua +++ b/src/apps/config/follower.lua @@ -33,7 +33,14 @@ function Follower:handle_actions_from_leader() local buf, len = channel:peek_message() if not buf then break end local action = action_codec.decode(buf, len) - app.apply_config_actions({action}) + local name, args = unpack(action) + if name == 'call_app_method_with_blob' then + local callee, method, blob = unpack(args) + local obj = assert(app.app_table[callee]) + assert(obj[method])(obj, blob) + else + app.apply_config_actions({action}) + end channel:discard_message(len) if action[1] == 'start_app' or action[1] == 'reconfig_app' then should_flush = true diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 38557efb3c..a1d33f74a7 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -257,7 +257,7 @@ local function path_adder_for_grammar(grammar, path) local ctab = getter(config) for entry in subconfig:iterate() do if ctab:lookup_ptr(entry.key) ~= nil then - error('already-existing entry', entry.key) + error('already-existing entry') end end for entry in subconfig:iterate() do diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index e5fb6c1a84..606281975b 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -1,9 +1,53 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) +local ffi = require('ffi') local app = require('core.app') +local lib = require('core.lib') +local data = require('lib.yang.data') +local yang = require('lib.yang.yang') +local path_mod = require('lib.yang.path') + +local function add_softwire_entry_actions(app_graph, entries) + assert(app_graph.apps['lwaftr']) + local ret = {} + for entry in entries:iterate() do + local blob = entries.entry_type() + ffi.copy(blob, entry, ffi.sizeof(blob)) + local args = {'lwaftr', 'add_softwire_entry', blob} + table.insert(ret, {'call_app_method_with_blob', args}) + end + return ret +end + +local softwire_grammar +local function get_softwire_grammar() + if not softwire_grammar then + local schema = yang.load_schema_by_name('snabb-softwire-v1') + local grammar = data.data_grammar_from_schema(schema) + softwire_grammar = + assert(grammar.members['binding-table'].members['softwire']) + end + return softwire_grammar +end + +local function remove_softwire_entry_actions(app_graph, path) + assert(app_graph.apps['lwaftr']) + path = path_mod.parse_path(path) + local grammar = get_softwire_grammar() + local key = path_mod.prepare_table_lookup( + grammar.keys, grammar.key_ctype, path[#path].query) + local args = {'lwaftr', 'remove_softwire_entry', key} + return {{'call_app_method_with_blob', args}} +end local function compute_config_actions(old_graph, new_graph, verb, path, arg) - return app.compute_config_actions(old_graph, new_graph) + if verb == 'add' and path == '/binding-table/softwire' then + return add_softwire_entry_actions(new_graph, arg) + elseif verb == 'remove' and path:match('^/binding%-table/softwire') then + return remove_softwire_entry_actions(new_graph, path) + else + return app.compute_config_actions(old_graph, new_graph) + end end function get_config_support() diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index 9f3c00ef7b..f388af1346 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -149,6 +149,20 @@ function BindingTable.new(psid_map, br_addresses, softwires) return setmetatable(ret, {__index=BindingTable}) end +function BindingTable:add_softwire_entry(entry_blob) + local entry = self.softwires.entry_type() + assert(ffi.sizeof(entry) == ffi.sizeof(entry_blob)) + ffi.copy(entry, entry_blob, ffi.sizeof(entry_blob)) + self.softwires:add(entry.key, entry.value) +end + +function BindingTable:remove_softwire_entry(entry_key_blob) + local entry = self.softwires.entry_type() + assert(ffi.sizeof(entry.key) == ffi.sizeof(entry_key_blob)) + ffi.copy(entry.key, entry_key_blob, ffi.sizeof(entry_key_blob)) + self.softwires:remove(entry.key) +end + local lookup_key = softwire_key_t() function BindingTable:lookup(ipv4, port) local psid = self:lookup_psid(ipv4, port) diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 0bb9f3cc74..7a26c96e9d 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -265,6 +265,16 @@ function LwAftr:new(conf) return o end +-- The following two methods are called by apps.config.follower in +-- reaction to binding table changes, via +-- apps/config/support/snabb-softwire-v1.lua. +function LwAftr:add_softwire_entry(entry_blob) + self.binding_table:add_softwire_entry(entry_blob) +end +function LwAftr:remove_softwire_entry(entry_key_blob) + self.binding_table:remove_softwire_entry(entry_key_blob) +end + local function decrement_ttl(pkt) local ipv4_header = get_ethernet_payload(pkt) local chksum = bnot(ntohs(rd16(ipv4_header + o_ipv4_checksum))) From 5ab2dfc4e5e40f2189c3ecf3ed45b79854078776 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 28 Nov 2016 17:02:10 +0100 Subject: [PATCH 333/631] Remove unused softwire support import --- src/apps/config/support/snabb-softwire-v1.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 606281975b..82a057ce1c 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -2,7 +2,6 @@ module(..., package.seeall) local ffi = require('ffi') local app = require('core.app') -local lib = require('core.lib') local data = require('lib.yang.data') local yang = require('lib.yang.yang') local path_mod = require('lib.yang.path') From be12aaaa0d196b26fe63ec51877dfc0b267f3ad5 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 29 Nov 2016 10:35:06 +0100 Subject: [PATCH 334/631] Support named lwaftr instances and update ps program This allows one to specify the name of a lwaftr from the yang configuration which claims the name in the lwaftr. It also adds support for this in the new `ps` command. It will now list the name (preferentially) or if none exists, the PID. It also introduces a flag (`-p` or `--pid`) which allows for overriding the default behavour of `ps`. The behavour when using the `-p` flag will only show the PID instead of the name. --- src/core/app.lua | 3 +- src/lib/yang/snabb-softwire-v1.yang | 8 +++++ src/program/lwaftr/setup.lua | 5 +++ .../lwaftr/tests/data/icmp_on_fail.conf | 1 + src/program/ps/README | 9 ++++-- src/program/ps/ps.lua | 32 ++++++++++++++++--- 6 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index 122fa312b3..fa96a06278 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -161,7 +161,8 @@ end -- This returns a table programs with the key being the name of the program -- and the value being the PID of the program. Each program is checked that -- it's still alive. Any dead program or program without a name is not listed. -function enumerate_named_programs() +-- If the "pidkey" is true, it will have the PID as the key instead of the name. +function enumerate_named_programs(pidkey) local progs = {} local dirs = shm.children("/by-name") if dirs == nil then return progs end diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index af11c212ba..d6d85eff0c 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -19,6 +19,14 @@ module snabb-softwire-v1 { description "Configuration for Snabb lwaftr."; + leaf name { + description + "Name of lwAFTR. This will be used as an identifier with programs within + snabb. This must be unique amongst other programs running on the + system."; + type string; + } + grouping traffic-filters { description "Ingress and egress filters describing the set of packets diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index af072850a7..53c6ce7087 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -1,5 +1,6 @@ module(..., package.seeall) +local engine = require("core.app") local config = require("core.config") local leader = require("apps.config.leader") local follower = require("apps.config.follower") @@ -100,6 +101,10 @@ function lwaftr_app(c, conf) set_preprocessors(c, preprocessing_apps_v6, "lwaftr.v6") set_postprocessors(c, "lwaftr.v6", postprocessing_apps_v6) set_postprocessors(c, "lwaftr.v4", postprocessing_apps_v4) + + if conf.name then + engine.claim_name(conf.name) + end end local function link_apps(c, apps) diff --git a/src/program/lwaftr/tests/data/icmp_on_fail.conf b/src/program/lwaftr/tests/data/icmp_on_fail.conf index 9490b763e4..4fc85afd50 100644 --- a/src/program/lwaftr/tests/data/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/icmp_on_fail.conf @@ -1,3 +1,4 @@ +name lwaftr1; binding-table { br-address 8:9:a:b:c:d:e:f; br-address 1e:1:1:1:1:1:1:af; diff --git a/src/program/ps/README b/src/program/ps/README index 16eebb2d0c..ef1db90734 100644 --- a/src/program/ps/README +++ b/src/program/ps/README @@ -3,6 +3,11 @@ Print a list of running Snabb instances. Available options: -h, --help Display this message. + -p, --PID Display processes by their PID instead of name. -The default output format is one PID per line. More options can be -added in the future. +The default output format is one identifier by line. If the --p or --pid flag is +used then the it will ignore any name defined and list just the PID. More +options can be added in the future. + +The identifier is either the name (preferentially) or, if one is not defined, +the PID of the process is used. diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index 6a695eb6f6..91036c35ac 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -5,6 +5,7 @@ module(..., package.seeall) local S = require("syscall") local lib = require("core.lib") local shm = require("core.shm") +local app = require("core.app") local function usage (code) local f = code == 0 and io.stdout or io.stderr @@ -14,26 +15,49 @@ end local function parse_args (args) local opt = {} + local preferpid = false function opt.h (arg) usage(0) end - args = lib.dogetopt(args, opt, "h", {help='h'}) + function opt.p (arg) preferpid = true end + args = lib.dogetopt(args, opt, "h:p", {help='h', pid='p'}) if #args ~= 0 then usage(1) end + return preferpid +end + +local function appname_resolver() + local apps = {} + for name, pid in pairs(app.enumerate_named_programs()) do + apps[pid] = name + end + return function (pid) return apps[pid] end end local function compute_snabb_instances() -- Produces set of snabb instances, excluding this one. + local whichname = appname_resolver() local pids = {} local my_pid = S.getpid() for _, name in ipairs(shm.children("/")) do -- This could fail as the name could be for example "by-name" local p = tonumber(name) - if p and p ~= my_pid then table.insert(pids, p) end + local name = whichname(p) + if p and p ~= my_pid then table.insert(pids, {pid=p, name=name}) end end return pids end function run (args) - parse_args (args) + local preferpid = parse_args (args) local instances = compute_snabb_instances() - for _, instance in ipairs(instances) do print(instance) end + for _, instance in ipairs(instances) do + if preferpid then + print(instance.pid) + else + if instance.name then + print(instance.name) + else + print(instance.pid) + end + end + end main.exit(0) end From 1ca6e6ead2690028ee3c34f75c3cbb0f3ea76f55 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 29 Nov 2016 15:46:09 +0100 Subject: [PATCH 335/631] Add name param to lwaftr run program --- src/program/lwaftr/bench/README | 3 +++ src/program/lwaftr/bench/bench.lua | 6 ++++-- src/program/lwaftr/run/README | 2 ++ src/program/lwaftr/run/run.lua | 7 +++++-- src/program/lwaftr/tests/data/icmp_on_fail.conf | 1 - src/program/ps/ps.lua | 6 +++--- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/program/lwaftr/bench/README b/src/program/lwaftr/bench/README index 59f7d909b9..8fb369a936 100644 --- a/src/program/lwaftr/bench/README +++ b/src/program/lwaftr/bench/README @@ -18,6 +18,9 @@ Usage: bench CONF IPV4-IN.PCAP IPV6-IN.PCAP is "bench.csv". -D DURATION, --duration DURATION Duration in seconds. + -n NAME, --name NAME Sets the name for this program, which will be used + as the identifier. This must be unique amongst + other snabb processes. Run the lwAFTR with input from IPV4-IN.PCAP and IPV6-IN.PCAP. The bench command is used to get an idea of the raw speed of the lwaftr without diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index fa55c63db9..36acba9d27 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -21,8 +21,8 @@ function parse_args(args) function handlers.b(arg) opts.bench_file = arg end function handlers.y() opts.hydra = true end function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "hyb:D:", { - help="h", hydra="y", ["bench-file"]="b", duration="D" }) + args = lib.dogetopt(args, handlers, "hyb:D:n", { + help="h", hydra="y", ["bench-file"]="b", duration="D", name="n"}) if #args ~= 3 then show_usage(1) end return opts, unpack(args) end @@ -31,6 +31,8 @@ function run(args) local opts, conf_file, inv4_pcap, inv6_pcap = parse_args(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) + if opts.name then engine.claim_name(opts.name) end + local graph = config.new() setup.reconfigurable(setup.load_bench, graph, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') diff --git a/src/program/lwaftr/run/README b/src/program/lwaftr/run/README index f2270065c5..24ed6c9f08 100644 --- a/src/program/lwaftr/run/README +++ b/src/program/lwaftr/run/README @@ -32,6 +32,8 @@ Optional arguments: written. A simple filename or relative pathname will be based on the current directory. Default is "bench.csv". + -n NAME, --name NAME Sets the name as the identifier of this program. + This must be unique amongst other snab programs. When the -v option is used at least once, packets on the network interfaces are counted and recorded, and the corresponding incoming and outgoing packet rates diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 5ac78cb5fe..ba322e87b3 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -25,6 +25,7 @@ function parse_args(args) verbosity = 0, ingress_drop_monitor = 'flush', bench_file = 'bench.csv' } local handlers = {} local cpu + function handlers.n (arg) opts.name = assert(arg) end function handlers.v () opts.verbosity = opts.verbosity + 1 end function handlers.i () opts.virtio_net = true end function handlers.D (arg) @@ -95,12 +96,12 @@ function parse_args(args) end function handlers.reconfigurable() opts.reconfigurable = true end function handlers.h() show_usage(0) end - lib.dogetopt(args, handlers, "b:c:vD:yhir:", + lib.dogetopt(args, handlers, "b:c:vD:yhir:n:", { conf = "c", v4 = 1, v6 = 1, ["v4-pci"] = 1, ["v6-pci"] = 1, verbose = "v", duration = "D", help = "h", virtio = "i", cpu = 1, ["ring-buffer-size"] = "r", ["real-time"] = 0, ["bench-file"] = "b", ["ingress-drop-monitor"] = 1, ["on-a-stick"] = 1, mirror = 1, - hydra = "y", reconfigurable = 0 }) + hydra = "y", reconfigurable = 0, name="n" }) if ring_buffer_size ~= nil then if opts.virtio_net then fatal("setting --ring-buffer-size does not work with --virtio") @@ -136,6 +137,8 @@ function run(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local use_splitter = requires_splitter(opts, conf) + if opts.name then engine.claim_name(otps.name) end + local c = config.new() local setup_fn, setup_args if opts.virtio_net then diff --git a/src/program/lwaftr/tests/data/icmp_on_fail.conf b/src/program/lwaftr/tests/data/icmp_on_fail.conf index 4fc85afd50..9490b763e4 100644 --- a/src/program/lwaftr/tests/data/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/icmp_on_fail.conf @@ -1,4 +1,3 @@ -name lwaftr1; binding-table { br-address 8:9:a:b:c:d:e:f; br-address 1e:1:1:1:1:1:1:af; diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index 91036c35ac..048c21259f 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -24,11 +24,11 @@ local function parse_args (args) end local function appname_resolver() - local apps = {} + local instances = {} for name, pid in pairs(app.enumerate_named_programs()) do - apps[pid] = name + instances[pid] = name end - return function (pid) return apps[pid] end + return function (pid) return instances[pid] end end local function compute_snabb_instances() From e8e3128bbdb166075d3ed9c4e5a9d3ef4b9511c4 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 29 Nov 2016 15:57:39 +0100 Subject: [PATCH 336/631] Remove name leaf from yang --- src/lib/yang/snabb-softwire-v1.yang | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index d6d85eff0c..af11c212ba 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -19,14 +19,6 @@ module snabb-softwire-v1 { description "Configuration for Snabb lwaftr."; - leaf name { - description - "Name of lwAFTR. This will be used as an identifier with programs within - snabb. This must be unique amongst other programs running on the - system."; - type string; - } - grouping traffic-filters { description "Ingress and egress filters describing the set of packets From ab8841bb517fba147f3e878f4aae5c4bd2520e1b Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 29 Nov 2016 16:21:14 +0100 Subject: [PATCH 337/631] Add handler for -n/--name for lwaftr bench --- src/program/lwaftr/bench/bench.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 36acba9d27..9b9cd43a2e 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -18,10 +18,11 @@ function parse_args(args) opts.duration = assert(tonumber(arg), "duration must be a number") assert(opts.duration >= 0, "duration can't be negative") end + function handlers.n(arg) opts.name = assert(arg) end function handlers.b(arg) opts.bench_file = arg end function handlers.y() opts.hydra = true end function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "hyb:D:n", { + args = lib.dogetopt(args, handlers, "n:hyb:D", { help="h", hydra="y", ["bench-file"]="b", duration="D", name="n"}) if #args ~= 3 then show_usage(1) end return opts, unpack(args) From 88defd3e81d1e5557ae552463c744a42b076a893 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 29 Nov 2016 16:27:54 +0100 Subject: [PATCH 338/631] Remove code to handle schema defined name for lwaftr --- src/program/lwaftr/setup.lua | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 53c6ce7087..af072850a7 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -1,6 +1,5 @@ module(..., package.seeall) -local engine = require("core.app") local config = require("core.config") local leader = require("apps.config.leader") local follower = require("apps.config.follower") @@ -101,10 +100,6 @@ function lwaftr_app(c, conf) set_preprocessors(c, preprocessing_apps_v6, "lwaftr.v6") set_postprocessors(c, "lwaftr.v6", postprocessing_apps_v6) set_postprocessors(c, "lwaftr.v4", postprocessing_apps_v4) - - if conf.name then - engine.claim_name(conf.name) - end end local function link_apps(c, apps) From b84b46a78afbc1cd3c95a5c20cea7cc3d1602bd2 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 30 Nov 2016 10:11:18 +0100 Subject: [PATCH 339/631] Add better documentation for config get-state --- src/program/config/get_state/README | 1 + src/program/config/get_state/README.inc | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) create mode 120000 src/program/config/get_state/README diff --git a/src/program/config/get_state/README b/src/program/config/get_state/README new file mode 120000 index 0000000000..d25b9bc34d --- /dev/null +++ b/src/program/config/get_state/README @@ -0,0 +1 @@ +README.inc \ No newline at end of file diff --git a/src/program/config/get_state/README.inc b/src/program/config/get_state/README.inc index ac151e9246..cf0661304f 100644 --- a/src/program/config/get_state/README.inc +++ b/src/program/config/get_state/README.inc @@ -1,11 +1,13 @@ -Data Format Usage: - snabb config get-state +Usage: snabb config get-state [OPTION]... ID PATH +Get the state for a Snabb network function. -This command takes in a instance identifier (a name or a PID) and a path to -inside the schema and will display the statistics at that path. The program -will find the conters specified under the path and match those to counters -defined in the apps. +Available options: + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + -h, --help Displays this message. -Example usage: +If the --schema argument is not provided, "snabb config" will ask the data +plane for its native schema. The result will be printed on standard output. - $ snabb config get-state lwaftr1 /softwire-state/ +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. From bb63b31a90c9b16d6f685dca8fc178d742bb4c7e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 1 Dec 2016 12:44:04 +0100 Subject: [PATCH 340/631] Fix parsing of bare YANG scalars --- src/lib/yang/data.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index dbd92bd83d..0cd7a35ce0 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -404,7 +404,7 @@ function data_parser_from_grammar(production) function top_parsers.scalar(production) local parse = value_parser(production.argument_type) return function(str, filename) - return parse(parser_mod.parse_string(str, filename)) + return parse(parser_mod.parse_string(str, filename), '[bare scalar]') end end return assert(top_parsers[production.type])(production) From 614ed2f5822db976b3eec80870825c769e4594ff Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 1 Dec 2016 12:46:50 +0100 Subject: [PATCH 341/631] Restart apps when configuration changes in place Sometimes you might want to update a configuration in place, then update your app graph to match the configuration. However Snabb's core logic to determine which apps need restart after a configuration change uses `lib.equal` to compare the old app initarg and the new app initarg, and that strategy doesn't work when we actually end up updating the old app initarg in place. To solve this, we build a set of embedded objects that might be updated in place, for each app in the app graph. Then, given a change to a configuration at a path, we determine which objects in that configuration might be updated in place. Any app referencing one of those maybe-updated-in-place objects will need a restart. There is also a shortcut for specific schemas. In this way if you know that, say, adding an entry to the binding table of a lwaftr won't cause other apps to need to restart, then in that case the schema support code can avoid needlessly restarting apps. Fixes https://github.com/Igalia/snabb/issues/602. --- src/apps/config/leader.lua | 47 +++-- src/apps/config/support.lua | 197 ++++++++++++++++++ src/apps/config/support/snabb-softwire-v1.lua | 39 +++- 3 files changed, 258 insertions(+), 25 deletions(-) create mode 100644 src/apps/config/support.lua diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 7baa68341e..92bc9c6de1 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -16,6 +16,7 @@ local app = require("core.app") local shm = require("core.shm") local app_graph = require("core.config") local action_codec = require("apps.config.action_codec") +local support = require("apps.config.support") local channel = require("apps.config.channel") Leader = { @@ -40,19 +41,6 @@ local function open_socket (file) return socket end -local generic_schema_config_support = { - compute_config_actions = function(old_graph, new_graph, verb, path, ...) - return app.compute_config_actions(old_graph, new_graph) - end -} - -local function load_schema_config_support(schema_name) - local mod_name = 'apps.config.support.'..schema_name:gsub('-', '_') - local success, support_mod = pcall(require, mod_name) - if success then return support_mod.get_config_support() end - return generic_schema_config_support -end - function Leader:new (conf) local ret = setmetatable({}, {__index=Leader}) ret.socket_file_name = conf.socket_file_name @@ -61,11 +49,10 @@ function Leader:new (conf) ret.socket_file_name = instance_dir..'/'..ret.socket_file_name end ret.schema_name = conf.schema_name - ret.support = load_schema_config_support(conf.schema_name) + ret.support = support.load_schema_config_support(conf.schema_name) ret.socket = open_socket(ret.socket_file_name) ret.peers = {} ret.setup_fn = conf.setup_fn - ret.current_app_graph = app_graph.new() ret.period = 1/conf.Hz ret.next_time = app.now() ret.followers = {} @@ -75,18 +62,23 @@ function Leader:new (conf) ret.rpc_callee = rpc.prepare_callee('snabb-config-leader-v1') ret.rpc_handler = rpc.dispatch_handler(ret, 'rpc_') - ret:reset_configuration(conf.initial_configuration) + ret:set_initial_configuration(conf.initial_configuration) return ret end -function Leader:reset_configuration (configuration) - local new_app_graph = self.setup_fn(configuration) +function Leader:set_initial_configuration (configuration) + self.current_configuration = configuration + self.current_app_graph = self.setup_fn(configuration) + self.current_in_place_dependencies = {} + self.current_in_place_dependencies = + self.support.update_mutable_objects_embedded_in_app_initargs ( + {}, self.current_app_graph, self.schema_name, 'load', '/', + self.current_configuration) + local initial_app_graph = app_graph.new() -- Empty. local actions = self.support.compute_config_actions( - self.current_app_graph, new_app_graph, 'load') + initial_app_graph, self.current_app_graph, {}, 'load') self:enqueue_config_actions(actions) - self.current_app_graph = new_app_graph - self.current_configuration = configuration end function Leader:take_follower_message_queue () @@ -95,9 +87,12 @@ function Leader:take_follower_message_queue () return actions end +local verbose = os.getenv('SNABB_LEADER_VERBOSE') and true + function Leader:enqueue_config_actions (actions) local messages = {} for _,action in ipairs(actions) do + if verbose then print('encode', action[1], unpack(action[2])) end local buf, len = action_codec.encode(action) table.insert(messages, { buf=buf, len=len }) end @@ -386,13 +381,21 @@ end function Leader:update_configuration (schema_name, update_fn, verb, path, ...) assert(schema_name == self.schema_name) + local to_restart = + self.support.compute_apps_to_restart_after_configuration_update ( + self.schema_name, self.current_configuration, verb, path, + self.current_in_place_dependencies) local new_config = update_fn(self.current_configuration, ...) local new_app_graph = self.setup_fn(new_config) local actions = self.support.compute_config_actions( - self.current_app_graph, new_app_graph, verb, path, ...) + self.current_app_graph, new_app_graph, to_restart, verb, path, ...) self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph self.current_configuration = new_config + self.current_in_place_dependencies = + self.support.update_mutable_objects_embedded_in_app_initargs ( + self.current_in_place_dependencies, self.current_app_graph, + verb, path, ...) end function Leader:handle_rpc_update_config (args, verb, compute_update_fn) diff --git a/src/apps/config/support.lua b/src/apps/config/support.lua new file mode 100644 index 0000000000..a3baa1582a --- /dev/null +++ b/src/apps/config/support.lua @@ -0,0 +1,197 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +local app = require("core.app") +local app_graph_mod = require("core.config") +local path_mod = require("lib.yang.path") +local yang = require("lib.yang.yang") +local data = require("lib.yang.data") +local cltable = require("lib.cltable") + +function compute_parent_paths(path) + local function sorted_keys(t) + local ret = {} + for k, v in pairs(t) do table.insert(ret, k) end + table.sort(ret) + return ret + end + local ret = { '/' } + local head = '' + for _,part in ipairs(path_mod.parse_path(path)) do + head = head..'/'..part.name + table.insert(ret, head) + local keys = sorted_keys(part.query) + if #keys > 0 then + for _,k in ipairs(keys) do + head = head..'['..k..'='..part.query[k]..']' + end + table.insert(ret, head) + end + end + return ret +end + +local function add_child_objects(accum, grammar, config) + local visitor = {} + local function visit(grammar, config) + assert(visitor[grammar.type])(grammar, config) + end + local function visit_child(grammar, config) + if grammar.type == 'scalar' then return end + table.insert(accum, config) + return visit(grammar, config) + end + function visitor.table(grammar, config) + -- Ctables are raw data, and raw data doesn't contain children + -- with distinct identity. + if grammar.key_ctype and grammar.value_ctype then return end + local child_grammar = {type="struct", members=grammar.values, + ctype=grammar.value_ctype} + if grammar.key_ctype then + for k, v in cltable.pairs(config) do visit_child(child_grammar, v) end + else + for k, v in pairs(config) do visit_child(child_grammar, v) end + end + end + function visitor.array(grammar, config) + -- Children are leaves; nothing to do. + end + function visitor.struct(grammar, config) + -- Raw data doesn't contain children with distinct identity. + if grammar.ctype then return end + for k,grammar in pairs(grammar.members) do + local id = data.normalize_id(k) + local child = config[id] + if child ~= nil then visit_child(grammar, child) end + end + end + return visit(grammar, config) +end + +local function compute_objects_maybe_updated_in_place (schema_name, config, + changed_path) + local schema = yang.load_schema_by_name(schema_name) + local grammar = data.data_grammar_from_schema(schema) + local objs = {} + local getter, subgrammar + for _,path in ipairs(compute_parent_paths(changed_path)) do + -- Calling the getter is avg O(N) in depth, so that makes the + -- loop O(N^2), though it is generally bounded at a shallow + -- level so perhaps it's OK. path_mod.resolver is O(N) too but + -- memoization makes it O(1). + getter, subgrammar = path_mod.resolver(grammar, path) + -- Scalars can't be updated in place. + if subgrammar.type == 'scalar' then return objs end + table.insert(objs, getter(config)) + -- Members of raw data can't be updated in place either. + if subgrammar.type == 'table' then + if subgrammar.key_ctype and subgrammar.value_ctype then return objs end + elseif subgrammar.type == 'struct' then + if subgrammar.ctype then return objs end + end + end + -- If the loop above finished normally, then it means that the + -- object at changed_path might contain in-place-updatable objects + -- inside of it, so visit its children. + add_child_objects(objs, subgrammar, objs[#objs]) + return objs +end + +local function record_mutable_objects_embedded_in_app_initarg (name, obj, accum) + local function record(obj) + local tab = accum[obj] + if not tab then tab = {}; accum[obj] = tab end + table.insert(tab, name) + end + local function visit(obj) + if type(obj) == 'table' then + record(obj) + for _,v in pairs(obj) do visit(v) end + elseif type(obj) == 'cdata' then + record(obj) + -- Cdata contains sub-objects but they don't have identity; + -- it's only the cdata object itself that has identity. + else + -- Other object kinds can't be updated in place. + end + end + visit(obj) +end + +-- Return "in-place dependencies": a table mapping mutable object -> +-- list of app names. +local function compute_mutable_objects_embedded_in_app_initargs (app_graph) + local deps = {} + for name, info in pairs(app_graph.apps) do + record_mutable_objects_embedded_in_app_initarg(name, info.arg, deps) + end + return deps +end + +local function compute_apps_to_restart_after_configuration_update ( + schema_name, configuration, verb, changed_path, in_place_dependencies) + local maybe_updated = compute_objects_maybe_updated_in_place( + schema_name, configuration, changed_path) + local needs_restart = {} + for _,place in ipairs(maybe_updated) do + for _,appname in ipairs(in_place_dependencies[place] or {}) do + needs_restart[appname] = true + end + end + return needs_restart +end + +local function add_restarts(actions, app_graph, to_restart) + for _,action in ipairs(actions) do + local name, args = unpack(action) + if name == 'stop_app' or name == 'reconfig_app' then + local appname = args[1] + to_restart[appname] = nil + end + end + local to_relink = {} + for appname, _ in pairs(to_restart) do + local info = assert(app_graph.apps[appname]) + local class, arg = info.class, info.arg + if class.reconfig then + table.insert(actions, {'reconfig_app', {appname, class, arg}}) + else + table.insert(actions, {'stop_app', {appname}}) + table.insert(actions, {'start_app', {appname, class, arg}}) + to_relink[appname] = true + end + end + for linkspec,_ in pairs(app_graph.links) do + local fa, fl, ta, tl = app_graph_mod.parse_link(linkspec) + if to_relink[fa] then + table.insert(actions, {'link_output', {fa, fl, linkspec}}) + end + if to_relink[ta] then + table.insert(actions, {'link_input', {ta, tl, linkspec}}) + end + end + return actions +end + + +generic_schema_config_support = { + compute_config_actions = function( + old_graph, new_graph, to_restart, verb, path, ...) + return add_restarts(app.compute_config_actions(old_graph, new_graph), + new_graph, to_restart) + end, + update_mutable_objects_embedded_in_app_initargs = function( + in_place_dependencies, app_graph, schema_name, verb, path, arg) + return compute_mutable_objects_embedded_in_app_initargs(app_graph) + end, + compute_apps_to_restart_after_configuration_update = + compute_apps_to_restart_after_configuration_update +} + +function load_schema_config_support(schema_name) + local mod_name = 'apps.config.support.'..schema_name:gsub('-', '_') + local success, support_mod = pcall(require, mod_name) + if success then return support_mod.get_config_support() end + return generic_schema_config_support +end diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 82a057ce1c..d88be3ca8b 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -5,6 +5,7 @@ local app = require('core.app') local data = require('lib.yang.data') local yang = require('lib.yang.yang') local path_mod = require('lib.yang.path') +local generic = require('apps.config.support').generic_schema_config_support local function add_softwire_entry_actions(app_graph, entries) assert(app_graph.apps['lwaftr']) @@ -39,16 +40,48 @@ local function remove_softwire_entry_actions(app_graph, path) return {{'call_app_method_with_blob', args}} end -local function compute_config_actions(old_graph, new_graph, verb, path, arg) +local function compute_config_actions(old_graph, new_graph, to_restart, + verb, path, arg) if verb == 'add' and path == '/binding-table/softwire' then return add_softwire_entry_actions(new_graph, arg) elseif verb == 'remove' and path:match('^/binding%-table/softwire') then return remove_softwire_entry_actions(new_graph, path) else - return app.compute_config_actions(old_graph, new_graph) + return generic.compute_config_actions( + old_graph, new_graph, to_restart, verb, path, arg) + end +end + +local function update_mutable_objects_embedded_in_app_initargs( + in_place_dependencies, app_graph, schema_name, verb, path, arg) + if verb == 'add' and path == '/binding-table/softwire' then + return in_place_dependencies + elseif verb == 'remove' and path:match('^/binding%-table/softwire') then + return in_place_dependencies + else + return generic.update_mutable_objects_embedded_in_app_initargs( + in_place_dependencies, app_graph, schema_name, verb, path, arg) + end +end + +local function compute_apps_to_restart_after_configuration_update( + schema_name, configuration, verb, path, in_place_dependencies) + if verb == 'add' and path == '/binding-table/softwire' then + return {} + elseif verb == 'remove' and path:match('^/binding%-table/softwire') then + return {} + else + return generic.compute_apps_to_restart_after_configuration_update( + schema_name, configuration, verb, path, in_place_dependencies) end end function get_config_support() - return { compute_config_actions = compute_config_actions } + return { + compute_config_actions = compute_config_actions, + update_mutable_objects_embedded_in_app_initargs = + update_mutable_objects_embedded_in_app_initargs, + compute_apps_to_restart_after_configuration_update = + compute_apps_to_restart_after_configuration_update + } end From 9139b8e65a3e412f782688e68b782734911919cc Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 28 Nov 2016 16:05:03 +0100 Subject: [PATCH 342/631] Make lwaftr bench multiprocess Note that this disables the spawn_csv_stats timer, as discussed last week. --- src/core/worker.lua | 1 + src/program/lwaftr/bench/bench.lua | 2 +- src/program/lwaftr/setup.lua | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index 17e2b28960..41f73ae287 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -50,6 +50,7 @@ function start (name, luacode) else -- Parent process children[name] = { pid = pid } + return pid end end diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 9b9cd43a2e..52a6969485 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -45,7 +45,7 @@ function run(args) csv:add_app('sinkv6', { 'input' }, { input=opts.hydra and 'encap' or 'Encap.' }) csv:activate() end - timer.activate(timer.new('spawn_csv_stats', start_sampling, 1e6)) + --timer.activate(timer.new('spawn_csv_stats', start_sampling, 1e6)) app.busywait = true app.main({duration=opts.duration}) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index af072850a7..0b9073357a 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -1,6 +1,7 @@ module(..., package.seeall) local config = require("core.config") +local worker = require("core.worker") local leader = require("apps.config.leader") local follower = require("apps.config.follower") local Intel82599 = require("apps.intel.intel_app").Intel82599 @@ -451,9 +452,20 @@ function reconfigurable(f, graph, conf, ...) f(graph, conf, unpack(args)) return graph end + + local worker_code = [[ + local follower = require("apps.config.follower") + local app = require("core.app") + local myconf = config.new() + config.app(myconf, "follower", follower.Follower, {}) + app.configure(myconf) + app.busywait = true + app.main({}) + ]] + local follower_pid = worker.start("follower", worker_code) + config.app(graph, 'leader', leader.Leader, { setup_fn = setup_fn, initial_configuration = conf, - follower_pids = { require('syscall').getpid() }, + follower_pids = { follower_pid }, schema_name = 'snabb-softwire-v1'}) - config.app(graph, "follower", follower.Follower, {}) end From 0f2d064c6c35a4217d4dac378b7e80ecba3a619a Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Thu, 1 Dec 2016 13:39:59 +0100 Subject: [PATCH 343/631] Add support for --cpu to bench The original process binds to the numa node associated with the specified CPU (if any), and the worker process binds to the specified CPU itself. --- src/program/lwaftr/bench/README | 1 + src/program/lwaftr/bench/bench.lua | 16 ++++++++++++++-- src/program/lwaftr/setup.lua | 13 +++++++++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/program/lwaftr/bench/README b/src/program/lwaftr/bench/README index 8fb369a936..652d9e0219 100644 --- a/src/program/lwaftr/bench/README +++ b/src/program/lwaftr/bench/README @@ -21,6 +21,7 @@ Usage: bench CONF IPV4-IN.PCAP IPV6-IN.PCAP -n NAME, --name NAME Sets the name for this program, which will be used as the identifier. This must be unique amongst other snabb processes. + --cpu Bind lwAFTR bench to the given CPU Run the lwAFTR with input from IPV4-IN.PCAP and IPV6-IN.PCAP. The bench command is used to get an idea of the raw speed of the lwaftr without diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 52a6969485..a5cfcb2e96 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -3,8 +3,10 @@ module(..., package.seeall) local app = require("core.app") local config = require("core.config") local lib = require("core.lib") +local numa = require("lib.numa") local csv_stats = require("program.lwaftr.csv_stats") local setup = require("program.lwaftr.setup") +local S = require("syscall") function show_usage(code) print(require("program.lwaftr.bench.README_inc")) @@ -18,12 +20,22 @@ function parse_args(args) opts.duration = assert(tonumber(arg), "duration must be a number") assert(opts.duration >= 0, "duration can't be negative") end + function handlers.cpu(arg) + cpu = tonumber(arg) + if not cpu or cpu ~= math.floor(cpu) or cpu < 0 then + fatal("Invalid cpu number: "..arg) + end + S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) + local wanted_node = numa.cpu_get_numa_node(cpu) + numa.bind_to_numa_node(wanted_node) + print("Bound to numa node: ", wanted_node) + end function handlers.n(arg) opts.name = assert(arg) end function handlers.b(arg) opts.bench_file = arg end function handlers.y() opts.hydra = true end function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "n:hyb:D", { - help="h", hydra="y", ["bench-file"]="b", duration="D", name="n"}) + args = lib.dogetopt(args, handlers, "n:hyb:D:", { + help="h", hydra="y", ["bench-file"]="b", duration="D", name="n", cpu=1 }) if #args ~= 3 then show_usage(1) end return opts, unpack(args) end diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 0b9073357a..e181a78d36 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -18,6 +18,7 @@ local vlan = require("apps.vlan.vlan") local ipv4 = require("lib.protocol.ipv4") local ethernet = require("lib.protocol.ethernet") local ipv4_ntop = require("lib.yang.util").ipv4_ntop +local S = require("syscall") local function convert_ipv4(addr) if addr ~= nil then return ipv4:pton(ipv4_ntop(addr)) end @@ -453,15 +454,23 @@ function reconfigurable(f, graph, conf, ...) return graph end - local worker_code = [[ + local worker_code = string.format([[ local follower = require("apps.config.follower") local app = require("core.app") + local numa = require("lib.numa") + + local target_cpu = tonumber(%s) + if target_cpu then + numa.bind_to_cpu(target_cpu) + print("Bound worker to CPU: ", target_cpu) + end local myconf = config.new() config.app(myconf, "follower", follower.Follower, {}) app.configure(myconf) app.busywait = true app.main({}) - ]] + ]], + S.getenv("SNABB_TARGET_CPU")) local follower_pid = worker.start("follower", worker_code) config.app(graph, 'leader', leader.Leader, From f078bcbe32243691eb2de74591fe7b31e4f08f95 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 1 Dec 2016 18:23:00 +0100 Subject: [PATCH 344/631] Fix problem in get-state causing number formatting issues There was a problem where the numbers where formatted with commas in to denote groups of three numbers. This was caused by the counters being wrapped in the core.counters_h struct. This fixe ensures they're unwrapped uint64 values. --- src/lib/yang/state.lua | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/yang/state.lua b/src/lib/yang/state.lua index 0f2f93a579..605eeb9eb7 100644 --- a/src/lib/yang/state.lua +++ b/src/lib/yang/state.lua @@ -23,14 +23,20 @@ local function flatten(val) return rtn end -local function find_counters(pid) +function find_counters(pid) local path = shm.root.."/"..pid..counter_directory - local counters = {} + local apps = {} for _, c in pairs(lib.files_in_directory(path)) do + local counters = {} local counterdir = "/"..pid..counter_directory.."/"..c - counters[c] = shm.open_frame(counterdir) + for k,v in pairs(shm.open_frame(counterdir)) do + if type(v) == "cdata" then + counters[k] = v.c + end + end + apps[c] = counters end - return counters + return apps end function collect_state_leaves(schema) From d48fe998bd2a9032c89b05113d462cf953a7aaaa Mon Sep 17 00:00:00 2001 From: Christian Graf Date: Thu, 1 Dec 2016 20:58:30 +0100 Subject: [PATCH 345/631] end-to-end section updated in case counters.lua file is used --- .../snabbvmx/doc/README.troubleshooting.md | 284 ++++++++++-------- 1 file changed, 151 insertions(+), 133 deletions(-) diff --git a/src/program/snabbvmx/doc/README.troubleshooting.md b/src/program/snabbvmx/doc/README.troubleshooting.md index 69ae0cfe38..4df9b1cee6 100644 --- a/src/program/snabbvmx/doc/README.troubleshooting.md +++ b/src/program/snabbvmx/doc/README.troubleshooting.md @@ -320,7 +320,34 @@ NOTE: Currently the test is not working correctly: the returned MAC should be `02:99:99:99:99:99`. -## End-to-end tests - details +## crafting and running end-to-end tests + +### brief steps + +(1) run the interactive check with "-r" parm to derive the counters and out.pcaps + +`snabb/src/snabb snabbvmx check -r ./CONF.cfg "./V4-IN.PCAP" "./V6-IN.PCAP" "./outv4.pcap" "./outv6.pcap" COUNTERS.lua` + +(2) place derived counters.lua in "snabb/src/program/snabbvmx/tests/end-to-end/data/counters" + +(3) place derived and expected out.pcaps in "snabb/src/program/snabbvmx/tests/end-to-end/data" + +(4) edit the "test_env.sh" in snabb/src/program/snabbvmx/tests/end-to-end to have the test scripted + +(5) run the scripted test: + +`snabb/src/program/snabbvmx/tests/end-to-end$ sudo ./end-to-end.sh` + + +### How the system test works in detail + +The way the test system works is by passing the input IPv4/IPv6 packets (via pre-recorded pcap-files) to the lwAFTR and comparing the expected packet output (pcap-file) to the packet output that the lwAFTR has generated. +The optional counters file is compared too to the actual counters file obtained after running the test. This is beeing reflected in snabbvmx check syntax: + +`CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP [COUNTERS.LUA]` + +V4-OUT.PCAP, V6-OUT.PCAP and COUNTERS.LUA are expected output. These output is compared to files stored temporarily in /tmp/endoutv4.pcap, /tmp/endoutv6.pcap and /tmp/counters.lua. These temporal files are the actual output produced by the lwAFTR after running a test. In order for a test to pass, the actual output must match the expected output. So it's not only that the counters file should match, but also the output .pcap files. (read: if a counters.lua file is provided, then it must still match the V4-OUT.PCAP and V6-OUT.PCAP) + **lwaftr vs snabbvmx** As both lwaftr and snabbvmx provide a different functionality and use different config-files, both the lwaftr and snabbvmx have their dedicated end-to-end tests. @@ -343,11 +370,14 @@ in Snabb's lwAFTR code, resulting in the addition of a new test to the lwAFTR's test suite. -**lwaftr** +**lwaftr** + ``` cd src/program/lwaftr/tests/end-to-end ``` -**snabbvmx** + +**snabbvmx** + ``` cd src/program/snabbvmx/tests/end-to-end ``` @@ -370,12 +400,12 @@ Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP Parameters: -- **CONF**: SnabbVMX (icmp_snabbvmx-lwaftr-xe.cfg) or lwaftr (icmp_snabbvmx-lwaftr-xe1.conf) configuration file. -- **V4-IN.PCAP**: Incoming IPv4 packets (from Internet). -- **V6-IN.PCAP**: Incoming IPv6 packets (from b4). -- **V4-OUT.PCAP**: Outgoing IPv4 packets (to Internet, decapsulated). -- **V6-OUT.PCAP**: Outgoing IPv6 packets (to b4, encapsulated) -- **[COUNTERS.LUA]**: Lua file with counter values. Will be regenerated via [-r] parm +- CONF: SnabbVMX (icmp_snabbvmx-lwaftr-xe.cfg) or lwaftr (icmp_snabbvmx-lwaftr-xe1.conf) configuration file. +- V4-IN.PCAP : Incoming IPv4 packets (from Internet). +- V6-IN.PCAP : Incoming IPv6 packets (from b4). +- V4-OUT.PCAP : Outgoing IPv4 packets (to Internet, decapsulated). +- V6-OUT.PCAP : Outgoing IPv6 packets (to b4, encapsulated) +- [COUNTERS.LUA] : Lua file with counter values. Will be regenerated via [-r] parm ## How to run SnabbVMX interactive end-to-end test @@ -386,8 +416,10 @@ configuration and binding table. With that information and knowing the error report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get decapsulated, etc), you craft a hand-made packet that meets the testing case. -**obtaining the config-files** -To run a test, the following config-files are required: +**obtaining the config-files** + +To run a test, the following config-files are required: + - the binding-table : binding_table.txt.s - lwaftr conf : snabbvmx-lwaftr-xe[0-9].conf - snabbvmx cfg : snabbvmx-lwaftr-xe[0-9].cfg @@ -401,7 +433,8 @@ or execute a shell within the dockers container and copy configs and binding-tab Note: the check application is just using a single interface. If the running container consists of two or more snabb-instances, then just take one of them for when running the check. -**collect-support-infos.sh** +**collect-support-infos.sh** + ``` lab@ubuntu1:~/vmxlwaftr/tests$ ./collect-support-infos.sh lwaftr3-16.2R3 collecting data in container lwaftr3-16.2R3 ... @@ -441,8 +474,10 @@ lab@ubuntu1:~/vmxlwaftr/tests/t1$ tar -tvzf support-info-20161108-1335.tgz -rwxr-xr-x root/root 2707019 2016-10-31 14:06 usr/local/bin/snabb ``` -**/tmp inside docker container** -The snabbvmx config-files can be derived from the container's shell as well directly +**config-files within /tmp inside docker container** + +The snabbvmx config-files can be derived from the container's shell as well directly within the /tmp directory + ``` lab@ubuntu1:~/vmxlwaftr/tests/t1$ docker exec -ti lwaftr3-16.2R3 bash pid 2654's current affinity mask: fffff @@ -456,9 +491,12 @@ binding_table.txt.s.o junos-vmx-x86-64-16.1-20160926.0.qcow2 pci_xe1 snabbv Note: press ctrl p ctrl q to exit the containers shell -**some adoption of config-files is required** -The advantage of snabbvmx is the dynamic next-hop resolution via Junos. When running the the lwaftr or snabbvmx app standalone, then the next-hop resolution via Junos is missing and this must be corrected. -**config as derived from a running vmxlwaftr container** +**some adoption of config-files is required** + +The advantage of snabbvmx is the dynamic next-hop resolution via Junos. When running the the lwaftr or snabbvmx app standalone, then the next-hop resolution via Junos is missing. The config-files are required to get modified for static next-hop configuration. + +**config as derived from a running vmxlwaftr container** + ``` lab@ubuntu1:~/vmxlwaftr/tests/t1$ cat snabbvmx-lwaftr-xe1.cfg return { @@ -478,13 +516,16 @@ return { } ``` -**Adopted config to use with vmxlwaftr** -Please make sure to change and add the below: +**Adopted static next-hop configuration to use with vmxlwaftr** + +To change configuration for static next-hop, below changes are required: + + - cache_refresh_interval = 0 (turns off next-hop learning via Junos) - mac_address (thats the own/self mac-address fo lwaftr) - next_hop_mac (next-hop mac to send the packets to) -Note: as seen, the snabbvmx config icmp_snabbvmx-lwaftr-xe1.cfg references the snabb-configuration file icmp_snabbvmx-lwaftr-xe1.conf. +Note: The snabbvmx config icmp_snabbvmx-lwaftr-xe1.cfg references the snabb-configuration file icmp_snabbvmx-lwaftr-xe1.conf via the "lwaftr" directive. ``` lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat icmp_snabbvmx-lwaftr-xe1.cfg @@ -509,16 +550,18 @@ return { } ``` -**The input pcaps** -The check app requires one or two input pcaps. -It is ok to: +**The input pcaps** + +The snabb "check" app requires one or two input pcaps. +It is ok to: + - only feed V4-IN.PCAP - only feed the V6-IN.PCAP - or feed both V4-IN.PCAP and V6-IN.PCAP Note for interactive tests: -When only feeding one pcap, then the other empty pcap must be the empty.pcap in -src/program/snabbvmx/tests/end-to-end/data/empty.pcap +When only feeding one pcap, then the other empty pcap must be the "empty.pcap" in +"src/program/snabbvmx/tests/end-to-end/data/empty.pcap" and not an empty string like "" ``` lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ ls empty.pcap @@ -529,9 +572,10 @@ lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/da reading from file empty.pcap, link-type EN10MB (Ethernet) ``` -**sample icmp-ipv4-in.pcap** -For below sample icmp-ipv4-in.pcap 6 * packets are seen. Only the 1st and 3rd frame is matching the binding-table and shall the encapsulated into lw4o6. The remaining 4 are not matching the binding-table. For those 4 packets the setting of policy_icmpv4_outgoing will define if the lwaftr will generate icmp dest unreachble or just silently discard the packets. -see lwaftr config-file: icmp_snabbvmx-lwaftr-xe1.conf. +**sample icmp-ipv4-in.pcap** + +For any input pcap it makes sense to keep it short - ideally a single packet to check correctness of the lwaftr. +The input packet "11:26:07.168372 IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain]" is matching the binding-table for PSID=1. ``` lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat cg-binding_table.txt.s @@ -550,24 +594,22 @@ softwires { { ipv4=10.10.0.0, psid=4, b4=2a02:587:f710::43 } } -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -r icmp-ipv4-in.pcap -e -reading from file icmp-ipv4-in.pcap, link-type EN10MB (Ethernet) -11:27:10.976929 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 42: 10.0.1.100.domain > 10.10.0.0.1024: [|domain] -11:27:10.988846 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 42: 10.0.1.100.domain > 10.10.0.0.domain: [|domain] -11:27:11.001278 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 242: 10.0.1.100 > 10.10.0.0: ICMP echo reply, id 1024, seq 0, length 208 -11:27:11.017448 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 242: 10.0.1.100 > 10.10.0.0: ICMP echo reply, id 53, seq 0, length 208 -11:27:11.029226 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 242: 192.168.0.0 > 192.168.1.100: ICMP echo reply, id 1024, seq 0, length 208 -11:27:11.040811 90:e2:ba:94:2a:bc (oui Unknown) > 02:cf:69:15:81:01 (oui Unknown), ethertype IPv4 (0x0800), length 42: 192.168.0.0.domain > 192.168.1.100.1024: [|domain] +lab@cgrafubuntu2:~$ tcpdump -n -r ipv4-in.pcap +reading from file ipv4-in.pcap, link-type EN10MB (Ethernet) +11:26:07.168372 IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] ``` -**running the interactive check** +**running the interactive check** +The interactive check is performed via `snabb snabbvmx check -r`. Using the "-r" parm instructs the check to generate the counters and out.pcap files. +As there is no IPv6 input, the "empty.pcap" is configured. + ``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ sudo ~/latest-snabb-binary/snabb/src/snabb snabbvmx check -r ./icmp_snabbvmx-lwaftr-xe1.cfg "./icmp-ipv4-in.pcap" "./empty.pcap" "./outv4/outv4.pcap" "./outv6/icmp-ipv6-out.pcap" icmp.lua -loading compiled binding table from ./cg-binding_table.txt.s.o -compiled binding table ./cg-binding_table.txt.s.o is up to date. +sudo ~/vmx-docker-lwaftr/snabb/src/snabb snabbvmx check -r ./snabbvmx-lwaftr-xe1.cfg "./ipv4-in.pcap" "./empty.pcap" "./outv4.pcap" "./outv6.pcap" test.lua +loading compiled binding table from ./binding_table.txt.s.o +compiled binding table ./binding_table.txt.s.o is up to date. nh_fwd4: cache_refresh_interval set to 0 seconds nh_fwd4: static next_hop_mac 90:e2:ba:94:2a:bc nh_fwd6: cache_refresh_interval set to 0 seconds @@ -577,78 +619,71 @@ done **results** -Check that your output matches what you expect: -For given input, the output is matching what we expect: -- 2 translated packets -- an empty outv4.pcap. lwaftr did not generate icmp dst-unreachable, because of the -```bash -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv6/icmp-ipv6-out.pcap -reading from file outv6/icmp-ipv6-out.pcap, link-type EN10MB (Ethernet) -01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::40: IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] -01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::40: IP 10.0.1.100 > 10.10.0.0: ICMP echo reply, id 1024, seq 0, length 208 +The -r flag is set, as such the resulting counters file test.lua and the out.pcap files are generated freshly. +The results below show a correct processing of the lwatr: -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv4/outv4.pcap -reading from file outv4/outv4.pcap, link-type EN10MB (Ethernet) -``` +- the counters-file lists one IPv4 packet as input +- as the input-packet matches the binding-table, one IPv6 packet output +- outv4.pcap is empty +- outv6.pcap shows the resulting encapsulated lw4o6 packet -**checking the counters** -The -r flag is set, as such the resulting counters file icmp.lua is generated freshly. The file does match our expectation: -- 2 * IPv6 packets out >> ["out-ipv6-packets"] = 2 -- 4 packets dropped becasue of the policy, as such outv4.pcap is an empty pcap-file ``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ grep policy_icmpv4_outgoing icmp_snabbvmx-lwaftr-xe1.conf -policy_icmpv4_outgoing = drop, -``` - -**cat icmp.lua** -``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat icmp.lua +lab@ubuntu1:~/vmx-docker-lwaftr/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat test.lua return { - ["drop-all-ipv4-iface-bytes"] = 568, - ["drop-all-ipv4-iface-packets"] = 4, - ["drop-no-dest-softwire-ipv4-bytes"] = 568, - ["drop-no-dest-softwire-ipv4-packets"] = 4, - ["drop-out-by-policy-icmpv4-packets"] = 4, - ["in-ipv4-bytes"] = 852, - ["in-ipv4-packets"] = 6, - ["out-ipv6-bytes"] = 364, - ["out-ipv6-packets"] = 2, + ["in-ipv4-bytes"] = 42, + ["in-ipv4-packets"] = 1, + ["out-ipv6-bytes"] = 82, + ["out-ipv6-packets"] = 1, } -``` +lab@ubuntu1:~/vmx-docker-lwaftr/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv4.pcap +reading from file outv4.pcap, link-type EN10MB (Ethernet) +lab@ubuntu1:~/vmx-docker-lwaftr/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv6.pcap +reading from file outv6.pcap, link-type EN10MB (Ethernet) +01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::400: IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] + +``` -**summary interactive check** +**summary interactive check** At this stage the interactive test is finished. -The following is defined: +The following is defined: + - lwaftr and snabbvmx configs - binding-table - input pcaps - -Further more, the interactive tests produced results, which is: -- expected out V4-OUT.PCAP and V6-OUT.PCAP (which can be empty) -- when using -r switch, then a counters-file got generated as well +- expected resulting counters and out.pcap With known input and results, the test can now be added to the scripted end-to-end.sh - to be executed with all other tests to ensure snabbvmx behaves as it should. -**Note:** -Further more, this procedure can be ideally used to report issues! - -The input-pcaps are Checking what values are in the counters can give you a hint about whether -things are working correctly or not. +Further more, this end-to-end procedure can be ideally used to report issues! Tip: packets always arrive only in one interface, but the output might be -empty for both interfaces, non-empty, and empty or non-empty for both cases. +empty or non-empty for both IPv4 and IPv6. + + +## adding the sample-test towards the scripted end-to-end tests +**in short** -## adding the icmp-test towards the scripted end-to-end tests +- place counter file and out.pcaps into correct directory +- edit "test_env.sh" and add the test +- run the test -**structure end-to-end tests** -**program/snabbvmx/tests/end-to-end/selftest.sh**: Runs end-to-end tests -(normal and VLAN). +**details** -The end-to-end tests are a test suite that tests the correctness of the lwAFTR -logic. +**directory-structure** +place out.pcap into the data-drectory + +``` +snabb/src/program/snabbvmx/tests/end-to-end/data +``` + +place counter-file into counters-drectory + +``` +snabb/src/program/snabbvmx/tests/end-to-end/data/counters +``` Files in **program/snabbvmx/tests/end-to-end/**: @@ -660,47 +695,20 @@ configuration files and binding tables. * **end-to-end-vlan.sh**: Runs **core-end-to-end.sh** on VLAN packets. * **selftest.sh**: Runs both **end-to-end.sh** and **end-to-end-vlan.sh** -**Note**: make sure all pcap and config-files are located in the data-drectory -``` -/home/lab/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data -``` -**Note**: make sure all counter-files counters-drectory -``` -/home/lab/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data/counters -``` +` -**adding the icmp-test**: +**adding the sample test**: -All tests are defined in the test_env.sh file. -This file has to be edited to include the icmp-test for example. -The check app has two criteria's to decide if the test is successful or not: -- check the counters-file -- check the resulting out.pcap file +All tests are defined in the "test_env.sh" file. +This file has to be edited to include the sample test +The check app always checks the provided data to decide if the test is successful or not: +- if the optional counters-file is provided, then it must match +- the resulting out.pcap files must always match -When running the interactive check with the -r option, then a "good" counters-file is created. This counters-file (here icmp.lua) can be referenced in the test_env.sh and if the actual test produced different values as defined in the icmp.lua file, then the test will fail -Another option is to rely on the outgoing pcaps. When running the interactive check, the resulting out.pcap files have been written. The test_env.sh can now run the same test and compare its pcap with the previous stored (./outv4/outv4.pcap , ./outv6/icmp-ipv6-out.pcap) files. If the files differ, then the test fails. +**pass-criteria without the optional counters.lua file** -**pass-criteria based on counters-file** ``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ vi test_env.sh -... -TEST_DATA=( - "ICMP Test - pass-critria out.pcap" - "icmp_snabbvmx-lwaftr-xe1.cfg" "icmp-ipv4-in.pcap" "" "" "" - "icmp.lua" -... - "IPv6 fragments and fragmentation is off" - "snabbvmx-lwaftr-xe1.cfg" "" "regressiontest-signedntohl-frags.pcap" "" "" - "drop-all-ipv6-fragments.lua" -) -``` -Running the end-to end.sh based on this configuration file shall not try to compare the out.pcap files. It shall only if the resulting counters are matching the previous defined icmp.lua counters. -If counters do match, then the test passed. - - -**pass-criteria based on pcap-file** -``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ vi test_env.sh +src/program/snabbvmx/tests/end-to-end$ vi test_env.sh ... # Contains an array of test cases. # @@ -711,8 +719,8 @@ lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ v # # Notice spaces and new lines are not taken into account. TEST_DATA=( - "ICMP Test - pass-critria out.pcap" - "icmp_snabbvmx-lwaftr-xe1.cfg" "icmp-ipv4-in.pcap" "empty.pcap" "" "icmp-ipv6-out.pcap" + "sample test" + "snabbvmx-lwaftr-xe1.cfg" "ipv4-in.pcap" "" "" "outv6.pcap" "" "IPv6 fragments and fragmentation is off" @@ -721,10 +729,10 @@ TEST_DATA=( ) -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ sudo ./end-to-end.sh -Testing: ICMP Test - pass-critria out.pcap -loading compiled binding table from data/cg-binding_table.txt.s.o -compiled binding table data/cg-binding_table.txt.s.o is up to date. +src/program/snabbvmx/tests/end-to-end$ sudo ./end-to-end.sh +Testing: sample test +loading compiled binding table from data/binding_table.txt.s.o +compiled binding table data/binding_table.txt.s.o is up to date. nh_fwd4: cache_refresh_interval set to 0 seconds nh_fwd4: static next_hop_mac 90:e2:ba:94:2a:bc nh_fwd6: cache_refresh_interval set to 0 seconds @@ -743,6 +751,16 @@ Test passed All end-to-end lwAFTR tests passed. ``` +If the counters file shall be taken into consideration as well (e.g. to count dropped frames), then just one line needs to be changed from "" to the counters file "test.lua" + +``` +lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ vi test_env.sh +... + "sample test" + "snabbvmx-lwaftr-xe1.cfg" "ipv4-in.pcap" "" "" "outv6.pcap" + "test.lua" +``` + From 51760554caf1574ec3948204560f39e5a932f65f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 11:03:18 +0100 Subject: [PATCH 346/631] Support alternate schemas in the leader The support code for the native schema can define translators for operations phrased in terms of a "foreign schema". It is assumed that anything representable in any foreign schema is also representable in the native schema: the native schema should be a superset of all foreign schemas. --- src/apps/config/leader.lua | 93 +++++++++++++++++-- src/apps/config/support.lua | 3 +- src/apps/config/support/snabb-softwire-v1.lua | 23 ++++- src/lib/yang/snabb-config-leader-v1.yang | 2 +- 4 files changed, 108 insertions(+), 13 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 92bc9c6de1..fe3aba1421 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -104,7 +104,12 @@ function Leader:enqueue_config_actions (actions) end function Leader:rpc_describe (args) - return { native_schema = self.schema_name } + local alternate_schemas = {} + for schema_name, translator in pairs(self.support.translators) do + table.insert(alternate_schemas, schema_name) + end + return { native_schema = self.schema_name, + alternate_schema = alternate_schemas } end local function path_printer_for_grammar(grammar, path) @@ -124,7 +129,9 @@ local function path_printer_for_schema_by_name(schema_name, path) end function Leader:rpc_get_config (args) - assert(args.schema == self.schema_name) + if args.schema ~= self.schema_name then + return self:foreign_rpc_get_config(args.schema, args.path) + end local printer = path_printer_for_schema_by_name(args.schema, args.path) local config = printer(self.current_configuration, yang.string_output_file()) return { config = config } @@ -380,7 +387,6 @@ function compute_remove_config_fn (schema_name, path) end function Leader:update_configuration (schema_name, update_fn, verb, path, ...) - assert(schema_name == self.schema_name) local to_restart = self.support.compute_apps_to_restart_after_configuration_update ( self.schema_name, self.current_configuration, verb, path, @@ -399,10 +405,6 @@ function Leader:update_configuration (schema_name, update_fn, verb, path, ...) end function Leader:handle_rpc_update_config (args, verb, compute_update_fn) - if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then - error('Attempt to modify configuration while listener attached') - end - assert(args.schema == self.schema_name) local path = path_mod.normalize_path(args.path) local parser = path_parser_for_schema_by_name(args.schema, path) self:update_configuration(args.schema, @@ -411,16 +413,86 @@ function Leader:handle_rpc_update_config (args, verb, compute_update_fn) return {} end +function Leader:get_translator (schema_name) + local translator = self.support.translators[schema_name] + if translator then return translator end + error('unsupported schema: '..schema_name) +end +function Leader:apply_translated_rpc_updates (updates) + for _,update in ipairs(updates) do + local verb, args = unpack(update) + local method = assert(self['rpc_'..verb..'_config']) + method(self, args) + end + return {} +end +function Leader:foreign_rpc_get_config (schema_name, path) + path = path_mod.normalize_path(path) + local translate = self:get_translator(schema_name) + local foreign_config = translate.get_config(self.current_configuration) + local printer = path_printer_for_schema_by_name(schema_name, path) + local config = printer(foreign_config, yang.string_output_file()) + return { config = config } +end +function Leader:foreign_rpc_get_state (schema_name, path) + path = path_mod.normalize_path(path) + local translate = self:get_translator(schema_name) + local native_state = state.show_state(self.schema_name, S.getpid(), "/") + local foreign_state = translate.get_state(native_state) + local printer = path_printer_for_schema_by_name(schema_name, path) + local config = printer(foreign_state, yang.string_output_file()) + return { state = config } +end +function Leader:foreign_rpc_set_config (schema_name, path, config_str) + path = path_mod.normalize_path(path) + local translate = self:get_translator(schema_name) + local parser = path_parser_for_schema_by_name(schema_name, path) + local updates = translate.set_config(self.current_configuration, path, + parser(config_str)) + return self:apply_translated_rpc_updates(updates) +end +function Leader:foreign_rpc_add_config (schema_name, path, config_str) + path = path_mod.normalize_path(path) + local translate = self:get_translator(schema_name) + local parser = path_parser_for_schema_by_name(schema_name, path) + local updates = translate.add_config(self.current_configuration, path, + parser(config_str)) + return self:apply_translated_rpc_updates(updates) +end +function Leader:foreign_rpc_remove_config (schema_name, path) + path = path_mod.normalize_path(path) + local translate = self:get_translator(schema_name) + local updates = translate.remove_config(self.current_configuration, path) + return self:apply_translated_rpc_updates(updates) +end + function Leader:rpc_set_config (args) + if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then + error('Attempt to modify configuration while listener attached') + end + if args.schema ~= self.schema_name then + return self:foreign_rpc_set_config(args.schema, args.path, args.config) + end return self:handle_rpc_update_config(args, 'set', compute_set_config_fn) end function Leader:rpc_add_config (args) + if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then + error('Attempt to modify configuration while listener attached') + end + if args.schema ~= self.schema_name then + return self:foreign_rpc_add_config(args.schema, args.path, args.config) + end return self:handle_rpc_update_config(args, 'add', compute_add_config_fn) end function Leader:rpc_remove_config (args) - assert(args.schema == self.schema_name) + if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then + error('Attempt to modify configuration while listener attached') + end + if args.schema ~= self.schema_name then + return self:foreign_rpc_remove_config(args.schema, args.path) + end local path = path_mod.normalize_path(args.path) self:update_configuration(args.schema, compute_remove_config_fn(args.schema, path), @@ -429,14 +501,15 @@ function Leader:rpc_remove_config (args) end function Leader:rpc_attach_listener (args) - assert(args.schema == self.schema_name) if self.listen_peer ~= nil then error('Listener already attached') end self.listen_peer = self.rpc_peer return {} end function Leader:rpc_get_state (args) - assert(args.schema == self.schema_name) + if args.schema ~= self.schema_name then + return self:foreign_rpc_get_state(args.schema, args.path) + end local printer = path_printer_for_schema_by_name(self.schema_name, args.path) local state = state.show_state(self.schema_name, S.getpid(), args.path) return {state=printer(state, yang.string_output_file())} diff --git a/src/apps/config/support.lua b/src/apps/config/support.lua index a3baa1582a..c93776818c 100644 --- a/src/apps/config/support.lua +++ b/src/apps/config/support.lua @@ -186,7 +186,8 @@ generic_schema_config_support = { return compute_mutable_objects_embedded_in_app_initargs(app_graph) end, compute_apps_to_restart_after_configuration_update = - compute_apps_to_restart_after_configuration_update + compute_apps_to_restart_after_configuration_update, + translators = {} } function load_schema_config_support(schema_name) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index d88be3ca8b..3c229331f9 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -76,12 +76,33 @@ local function compute_apps_to_restart_after_configuration_update( end end +local function ietf_softwire_translator () + local ret = {} + function ret.get_config(native_config) + error('unimplemented') + end + function ret.get_state(native_state) + error('unimplemented') + end + function ret.set_config(native_config, path, data) + error('unimplemented') + end + function ret.add_config(native_config, path, data) + error('unimplemented') + end + function ret.remove_config(native_config, path) + error('unimplemented') + end + return ret +end + function get_config_support() return { compute_config_actions = compute_config_actions, update_mutable_objects_embedded_in_app_initargs = update_mutable_objects_embedded_in_app_initargs, compute_apps_to_restart_after_configuration_update = - compute_apps_to_restart_after_configuration_update + compute_apps_to_restart_after_configuration_update, + translators = { ['ietf-softwire'] = ietf_softwire_translator () } } end diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index 4f13a7d455..9b4d201e0e 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -15,7 +15,7 @@ module snabb-config-leader-v1 { rpc describe { output { leaf native-schema { type string; mandatory true; } - // leaf-list alternate-schema { type string; } + leaf-list alternate-schema { type string; } } } From 55409bc0f9c627fd26c813357d6d43495f5712b0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 12:26:17 +0100 Subject: [PATCH 347/631] container YANG schema nodes always appear in data It used to be that the Snabb YANG support would expect that "container" schema nodes would only have corresponding data nodes if they had a "presence true;" attribute. This was a misunderstanding of how YANG expects to map schema to data. In particular this is a legal schema: container foo { leaf bar { type string; } } container baz { leaf bar { type string; } } That is to say, it's perfectly fine for there to be two leaves with the same name; they don't conflict because the corresponding data would look like: foo { bar "hey"; } baz { bar "ho"; } Before in Snabb we would expect data that looks like: bar "hey"; bar "ho"; And obviously that doesn't work, as you can't tell the leaves apart. This change adds a "--from" option to "snabb lwaftr migrate-configuration", indicating the version from which to migrate. The default is still "legacy", as before, but now there is a new version, --from=3.0.1, to migrate from the previous mapping between schema and data. This change updates all configurations in the source, removes "presence true" from example and test schemas where it's not necessary, and also changes to put the binding-table node inside softwire-config. --- src/lib/yang/README.md | 31 +- src/lib/yang/binary.lua | 1 - src/lib/yang/data.lua | 2 - src/lib/yang/path.lua | 2 - src/lib/yang/snabb-softwire-v1.yang | 251 +-- src/lib/yang/state.lua | 8 +- src/program/config/README.md | 4 - .../lwaftr/doc/README.configuration.md | 165 +- .../lwaftr/migrate_configuration/README | 14 +- .../migrate_configuration.lua | 24 +- .../lwaftr/tests/data/big_mtu_no_icmp.conf | 246 ++- .../lwaftr/tests/data/icmp_on_fail.conf | 238 +- src/program/lwaftr/tests/data/no_hairpin.conf | 244 +- src/program/lwaftr/tests/data/no_icmp.conf | 242 +- .../lwaftr/tests/data/no_icmp_maxfrags1.conf | 244 +- .../data/no_icmp_with_filters_accept.conf | 250 +-- .../no_icmp_with_filters_and_vlan_accept.conf | 254 +-- .../no_icmp_with_filters_and_vlan_drop.conf | 254 +-- .../tests/data/no_icmp_with_filters_drop.conf | 250 +-- .../tests/data/small_ipv4_mtu_icmp.conf | 236 +- .../tests/data/small_ipv6_mtu_no_icmp.conf | 246 ++- .../data/small_ipv6_mtu_no_icmp_allow.conf | 238 +- .../small_ipv6_mtu_no_icmp_vlan_allow.conf | 242 +- .../lwaftr/tests/data/tunnel_icmp.conf | 234 +- .../tests/data/tunnel_icmp_without_mac4.conf | 234 +- .../tests/data/tunnel_icmp_withoutmac.conf | 234 +- src/program/lwaftr/tests/data/vlan.conf | 238 +- .../tests/data/vlan/big_mtu_no_icmp.conf | 250 +-- .../lwaftr/tests/data/vlan/icmp_on_fail.conf | 242 +- .../lwaftr/tests/data/vlan/no_hairpin.conf | 248 +-- .../lwaftr/tests/data/vlan/no_icmp.conf | 246 ++- .../tests/data/vlan/no_icmp_maxfrags1.conf | 248 +-- .../vlan/no_icmp_with_filters_accept.conf | 254 +-- .../data/vlan/no_icmp_with_filters_drop.conf | 254 +-- .../tests/data/vlan/small_ipv4_mtu_icmp.conf | 240 +- .../data/vlan/small_ipv6_mtu_no_icmp.conf | 250 +-- .../vlan/small_ipv6_mtu_no_icmp_allow.conf | 242 +- .../lwaftr/tests/data/vlan/tunnel_icmp.conf | 238 +- .../data/vlan/tunnel_icmp_without_mac4.conf | 238 +- .../data/vlan/tunnel_icmp_withoutmac.conf | 238 +- src/program/lwaftr/tests/data/vlan/vlan.conf | 238 +- .../tests/conf/snabbvmx-lwaftr-xe0.conf | 1968 +++++++++-------- .../snabbvmx/tests/conf/snabbvmx-lwaftr.conf | 1968 +++++++++-------- .../end-to-end/data/snabbvmx-lwaftr-xe1.conf | 120 +- .../data/vlan/snabbvmx-lwaftr-xe1.conf | 124 +- 45 files changed, 6145 insertions(+), 6087 deletions(-) diff --git a/src/lib/yang/README.md b/src/lib/yang/README.md index 7e8fe2acde..c1c0baf75c 100644 --- a/src/lib/yang/README.md +++ b/src/lib/yang/README.md @@ -24,7 +24,6 @@ module snabb-simple-router { leaf active { type boolean; default true; } container routes { - presence true; list route { key addr; leaf addr { type inet:ipv4-address; mandatory true; } @@ -81,13 +80,9 @@ syntax from the schema, in the following way: is the name of the leaf, and the value is in the right syntax for the leaf's type. (More on value types below.) -- A `container`'s configuration can be one of two ways. Firstly, if - its `presence` attribute is `true`, then the container's - configuration is the container's keyword followed by the - configuration of its data node children, like `keyword { - configuration... }`. Otherwise if its `presence` is `false`, then a - container's configuration is just its data node children's - configuration, in any order. +- A `container`'s configuration is the container's keyword followed by + the configuration of its data node children, like `keyword { + configuration... }`. - A `leaf-list`'s configuration is a sequence of 0 or more instances of `keyword value;`, as in `leaf`. @@ -125,10 +120,7 @@ routes { Except in special cases as described in RFC 6020, order is insignificant. You could have `active false;` at the end, for example, and `route { addr 1.2.3.4; port 1; }` is the same as `route { -port 1; addr 1.2.3.4; }`. Note that if `presence` is false (the -default), the grammar is the same except there's no outer `routes { }` -wrapper; the `route` statements would be at the same level as -`active`. +port 1; addr 1.2.3.4; }`. The surface syntax of our configuration format is the same as for YANG schemas; `"1.2.3.4"` is the same as `1.2.3.4`. Snabb follows the XML @@ -260,9 +252,6 @@ In this case, the result would be a table with two keys, `active` and `routes`. The value of the `active` key would be Lua boolean `true`. The `routes` container is just another table of the same kind. -(Remember however that only containers with `presence true;` have -corresponding nodes in the configuration syntax, and corresponding -sub-tables in the result configuration objects.) Inside the `routes` container is the `route` list, which is represented as an associative array. The particular representation for the @@ -301,13 +290,13 @@ Let us return to the representation of compound configurations, like compiled to raw FFI data. A configuration's shape is determined by its schema. A schema node whose data will be fixed is either a leaf whose type is numeric or boolean and which is either mandatory or has a -default value, or a container (`leaf-list`, `container` with presence, -or `list`) whose elements are all themselves fixed. +default value, or a container (`leaf-list`, `container`, or `list`) +whose elements are all themselves fixed. -In practice this means that a fixed `container` with presence will be -compiled to an FFI `struct` type. This is mostly transparent from the -user perspective, as in LuaJIT you access struct members by name in the -same way as for normal Lua tables. +In practice this means that a fixed `container` will be compiled to an +FFI `struct` type. This is mostly transparent from the user +perspective, as in LuaJIT you access struct members by name in the same +way as for normal Lua tables. A fixed `leaf-list` will be compiled to an FFI array of its element type, but on the Lua side is given the normal 1-based indexing and diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 26df481745..fd7368d094 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -435,7 +435,6 @@ function selftest() leaf-list integers { type uint32; } leaf-list addrs { type inet:ipv4-address; } container routes { - presence true; list route { key addr; leaf addr { type inet:ipv4-address; mandatory true; } diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 0cd7a35ce0..e7702499d8 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -78,7 +78,6 @@ function data_grammar_from_schema(schema) end function handlers.container(node) local members = visit_body(node) - if not node.presence then return members end return {[node.id]={type='struct', members=members, ctype=struct_ctype(members)}} end @@ -670,7 +669,6 @@ function selftest() } container fruit-bowl { - presence true; leaf description { type string; } list contents { uses fruit; key name; } } diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 95d08e9d5b..50d9dffaa4 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -256,7 +256,6 @@ function selftest() leaf-list blocked-ips { type inet:ipv4-address; } container routes { - presence true; list route { key addr; leaf addr { type inet:ipv4-address; mandatory true; } @@ -315,7 +314,6 @@ function selftest() import ietf-inet-types {prefix inet;} container bowl { - presence true; list fruit { key name; leaf name { type string; mandatory true; } diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index af11c212ba..27e49b9483 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -80,8 +80,6 @@ module snabb-softwire-v1 { transmission."; container error-rate-limiting { - presence true; - leaf packets { type uint32; description @@ -104,8 +102,6 @@ module snabb-softwire-v1 { fragmented packets."; container reassembly { - presence true; - leaf max-fragments-per-packet { type uint32 { range 1..max; } default 20; @@ -137,8 +133,6 @@ module snabb-softwire-v1 { "Configuration for the external, internet-facing IPv4 interface."; - presence true; - leaf ip { type inet:ipv4-address; mandatory true; @@ -167,7 +161,6 @@ module snabb-softwire-v1 { uses reassembly; container next-hop { - presence true; leaf ip { type inet:ipv4-address; description @@ -187,8 +180,6 @@ module snabb-softwire-v1 { description "Configuration for the internal IPv6 interface."; - presence true; - leaf ip { type inet:ipv6-address; mandatory true; @@ -217,7 +208,6 @@ module snabb-softwire-v1 { uses reassembly; container next-hop { - presence true; leaf ip { type inet:ipv6-address; description @@ -240,149 +230,146 @@ module snabb-softwire-v1 { two B4s."; } } - } - container binding-table { - description - "A collection of softwires (tunnels), along with a description - of the IPv4 and IPv6 addresses handled by the lwAFTR."; - - presence true; - - list psid-map { + container binding-table { description - "The set of IPv4 addresses managed by the lwAFTR, along with - the way in which those IPv4 addresses share ports. A PSID map - entry associates a PSID length, shift, and - reserved-ports-bit-count with each IPv4 address served by - the lwAFTR. - - The lightweight 4-over-6 architecture supports sharing of - IPv4 addresses by partitioning the space of TCP/UDP/ICMP - ports into disjoint \"port sets\". Each softwire associated - with an IPv4 address corresponds to a different set of ports - on that address. The way that the ports are partitioned is - specified in RFC 7597: each address has an associated set - of parameters that specifies how to compute a \"port set - identifier\" (PSID) from a given port. - - 0 1 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - +-----------+-----------+-------+ - Ports in | A | PSID | j | - the CE port set | > 0 | | | - +-----------+-----------+-------+ - | a bits | k bits |m bits | - - Figure 2: Structure of a Port-Restricted Port Field - - Source: http://tools.ietf.org/html/rfc7597#section-5.1 - - We find the specification's names to be a bit obtuse, so we - refer to them using the following names: - - a bits = reserved-ports-bit-count. - k bits = psid-length. - m bits = shift."; - - key addr; - - leaf addr { - type inet:ipv4-address; - mandatory true; - description - "Public IPv4 address managed by the lwAFTR."; - } + "A collection of softwires (tunnels), along with a description + of the IPv4 and IPv6 addresses handled by the lwAFTR."; - leaf end-addr { - type inet:ipv4-address; + list psid-map { description - "If present, this PSID map entry applies to all addresses - between 'addr' and this address, inclusive."; - } + "The set of IPv4 addresses managed by the lwAFTR, along with + the way in which those IPv4 addresses share ports. A PSID map + entry associates a PSID length, shift, and + reserved-ports-bit-count with each IPv4 address served by + the lwAFTR. + + The lightweight 4-over-6 architecture supports sharing of + IPv4 addresses by partitioning the space of TCP/UDP/ICMP + ports into disjoint \"port sets\". Each softwire associated + with an IPv4 address corresponds to a different set of ports + on that address. The way that the ports are partitioned is + specified in RFC 7597: each address has an associated set + of parameters that specifies how to compute a \"port set + identifier\" (PSID) from a given port. + + 0 1 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + +-----------+-----------+-------+ + Ports in | A | PSID | j | + the CE port set | > 0 | | | + +-----------+-----------+-------+ + | a bits | k bits |m bits | + + Figure 2: Structure of a Port-Restricted Port Field + + Source: http://tools.ietf.org/html/rfc7597#section-5.1 + + We find the specification's names to be a bit obtuse, so we + refer to them using the following names: + + a bits = reserved-ports-bit-count. + k bits = psid-length. + m bits = shift."; + + key addr; + + leaf addr { + type inet:ipv4-address; + mandatory true; + description + "Public IPv4 address managed by the lwAFTR."; + } - leaf psid-length { - type uint8 { range 0..16; } - mandatory true; - description - "The number of bits devoted to the PSID in the port map. - If the psid-length is N, then the IPv4 address will be - shared 2^N ways. Note that psid-length, shift, and - reserved-ports-bit-count must add up to 16."; - } + leaf end-addr { + type inet:ipv4-address; + description + "If present, this PSID map entry applies to all addresses + between 'addr' and this address, inclusive."; + } - leaf shift { - type uint8 { range 0..16; } - description - "Given an incoming port, one can obtain the PSID by - shifting the port right by 'shift' bits and then masking - off the lowest 'psid-length' bits. Defaults to 16 - - psid-length. Note that psid-length, shift, and - reserved-ports-bit-count must add up to 16."; + leaf psid-length { + type uint8 { range 0..16; } + mandatory true; + description + "The number of bits devoted to the PSID in the port map. + If the psid-length is N, then the IPv4 address will be + shared 2^N ways. Note that psid-length, shift, and + reserved-ports-bit-count must add up to 16."; + } + + leaf shift { + type uint8 { range 0..16; } + description + "Given an incoming port, one can obtain the PSID by + shifting the port right by 'shift' bits and then masking + off the lowest 'psid-length' bits. Defaults to 16 - + psid-length. Note that psid-length, shift, and + reserved-ports-bit-count must add up to 16."; + } + + leaf reserved-ports-bit-count { + type uint8 { range 0..16; } + default 0; + description + "Reserve the lowest 2^N ports so that they map to no + softwire. This can be useful to prevent the low 1024 + ports (for example) from being mapped to customers. Note + that psid-length and shift must add up to less than or + equal to 16."; + } } - leaf reserved-ports-bit-count { - type uint8 { range 0..16; } - default 0; + leaf-list br-address { + type inet:ipv6-address; description - "Reserve the lowest 2^N ports so that they map to no - softwire. This can be useful to prevent the low 1024 - ports (for example) from being mapped to customers. Note - that psid-length and shift must add up to less than or - equal to 16."; + "B4-facing address of an lwAFTR."; } - } - - leaf-list br-address { - type inet:ipv6-address; - description - "B4-facing address of an lwAFTR."; - } - list softwire { - key "ipv4 psid padding"; + list softwire { + key "ipv4 psid padding"; - leaf ipv4 { - type inet:ipv4-address; - mandatory true; - description - "Public IPv4 address of the softwire."; - } + leaf ipv4 { + type inet:ipv4-address; + mandatory true; + description + "Public IPv4 address of the softwire."; + } - leaf psid { - type uint16; - mandatory true; - description - "Port set ID."; - } + leaf psid { + type uint16; + mandatory true; + description + "Port set ID."; + } - leaf padding { - type uint16 { range 0..0; } - default 0; - description - "Reserved bytes."; - } + leaf padding { + type uint16 { range 0..0; } + default 0; + description + "Reserved bytes."; + } - leaf br { - type uint32; - default 0; - description - "The B4-facing address of the lwAFTR for this softwire, as - a zero-based index into br-addresses."; - } + leaf br { + type uint32; + default 0; + description + "The B4-facing address of the lwAFTR for this softwire, as + a zero-based index into br-addresses."; + } - leaf b4-ipv6 { - type inet:ipv6-address; - mandatory true; - description - "B4 address."; + leaf b4-ipv6 { + type inet:ipv6-address; + mandatory true; + description + "B4 address."; + } } } } container softwire-state { description "State data about lwaftr."; - presence true; config false; /* Decapsulation B4 queue */ @@ -673,4 +660,4 @@ module snabb-softwire-v1 { setting max_ipv6_reassembly_packets."; } } -} \ No newline at end of file +} diff --git a/src/lib/yang/state.lua b/src/lib/yang/state.lua index 0f2f93a579..5cd456e2e5 100644 --- a/src/lib/yang/state.lua +++ b/src/lib/yang/state.lua @@ -115,19 +115,14 @@ function selftest () leaf-list blocked-ips { type inet:ipv4-address; } container routes { - presence true; list route { key addr; leaf addr { type inet:ipv4-address; mandatory true; } leaf port { type uint8 { range 0..11; } mandatory true; } } - } - - container state { - presence true; config false; leaf total-packets { @@ -153,7 +148,6 @@ function selftest () } container detailed-state { - presence true; config false; uses "detailed-counters"; } @@ -190,4 +184,4 @@ function selftest () local multi_dimentional = {{hello="hello"}, {world="world"}} assert(flatten(multi_dimentional), {hello="hello", world="world"}) print("selftest: ok") -end \ No newline at end of file +end diff --git a/src/program/config/README.md b/src/program/config/README.md index fc726b190e..668bf4f68d 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -71,7 +71,6 @@ module snabb-simple-router { leaf-list public-ip { type inet:ipv4-address; } container routes { - presence true; list route { key addr; leaf addr { type inet:ipv4-address; mandatory true; } @@ -104,9 +103,6 @@ The surface syntax of data is the same as for YANG schemas; you can have end-of-line comments with `//`, larger comments with `/* ... */`, and the YANG schema quoting rules for strings apply. -Note that containers like `route {}` only appear in the data syntax if -they are marked as `presence true;` in the schema. - So indeed, `snabb config get ID /` might print out just the output given above. diff --git a/src/program/lwaftr/doc/README.configuration.md b/src/program/lwaftr/doc/README.configuration.md index 808e82f0c5..1429b63154 100644 --- a/src/program/lwaftr/doc/README.configuration.md +++ b/src/program/lwaftr/doc/README.configuration.md @@ -10,88 +10,90 @@ file. That file's grammar is derived from the YANG schema; see the Here's an example: ``` -// IPv4 interface. -external-interface { - allow-incoming-icmp true; - // Limit ICMP error transmit rate to PACKETS/PERIOD. - error-rate-limiting { - packets 600000; - period 2; +softwire-config { + // IPv4 interface. + external-interface { + allow-incoming-icmp true; + // Limit ICMP error transmit rate to PACKETS/PERIOD. + error-rate-limiting { + packets 600000; + period 2; + } + // Generate ICMP errors at all? + generate-icmp-errors true; + // Basic parameters. + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1460; + // vlan-tag 42; + // Where to go next. Either one will suffice; if you specify the IP, + // the next-hop MAC will be determined by ARP. + next-hop { + mac 68:68:68:68:68:68; + ip 1.2.3.4; + } + // Control the size of the fragment reassembly buffer. + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } } - // Generate ICMP errors at all? - generate-icmp-errors true; - // Basic parameters. - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1460; - // vlan-tag 42; - // Where to go next. Either one will suffice; if you specify the IP, - // the next-hop MAC will be determined by ARP. - next-hop { - mac 68:68:68:68:68:68; - ip 1.2.3.4; + // The same thing as for the external interface, but on the IPv6 side + // and with IPv6 addresses. + internal-interface { + allow-incoming-icmp true; + error-rate-limiting { + packets 600000; + period 2; + } + generate-icmp-errors true; + // One more interesting thing -- here we control whether to support + // routing traffic between two B4s. + hairpinning true; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1500; + // vlan-tag 64; + next-hop { + mac 44:44:44:44:44:44; + // NDP instead of ARP of course. + ip 7:8:9:a:b:c:d:e; + } + reassembly { + max-fragments-per-packet 40; + max-packets 20000; + } } - // Control the size of the fragment reassembly buffer. - reassembly { - max-fragments-per-packet 40; - max-packets 20000; - } -} -// The same thing as for the external interface, but on the IPv6 side -// and with IPv6 addresses. -internal-interface { - allow-incoming-icmp true; - error-rate-limiting { - packets 600000; - period 2; - } - generate-icmp-errors true; - // One more interesting thing -- here we control whether to support - // routing traffic between two B4s. - hairpinning true; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 1500; - // vlan-tag 64; - next-hop { - mac 44:44:44:44:44:44; - // NDP instead of ARP of course. - ip 7:8:9:a:b:c:d:e; - } - reassembly { - max-fragments-per-packet 40; - max-packets 20000; - } -} -// Now the binding table! 3 parts: PSID map, BR address table, and -// softwire set. See description below for details. -binding-table { - psid-map { addr 178.79.150.15; psid-length 4; } - psid-map { addr 178.79.150.233; psid-length 16; } - psid-map { addr 178.79.150.2; psid-length 16; } - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + // Now the binding table! 3 parts: PSID map, BR address table, and + // softwire set. See description below for details. + binding-table { + psid-map { addr 178.79.150.15; psid-length 4; } + psid-map { addr 178.79.150.233; psid-length 16; } + psid-map { addr 178.79.150.2; psid-length 16; } + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } } } ``` @@ -100,7 +102,8 @@ Basically there's an `external-interface` section defining the parameters around the IPv4 interface that communicates with the internet, an `internal-interface` section doing the same for the IPv6 side that communicates with the B4s, and then the `binding-table` that -declares the set of softwires. +declares the set of softwires. The whole thing is surrounded in a +`softwire-config { ... }` block. ## Compiling conigurations diff --git a/src/program/lwaftr/migrate_configuration/README b/src/program/lwaftr/migrate_configuration/README index 0f5f9e33a7..a94efe8a8a 100644 --- a/src/program/lwaftr/migrate_configuration/README +++ b/src/program/lwaftr/migrate_configuration/README @@ -2,10 +2,20 @@ Usage: migrate-configuration LWAFTR.CONF Options: -h, --help Print usage information. + -f, --from=VERSION Specify version from which to migrate. Migrate an old-style configuration and binding table to the new YANG configuration. LWAFTR.CONF should be the name of an old lwAFTR -configuration. +configuration. Available VERSION values are: -The resulting new-style configuration will be printed on standard + legacy + Configuration from pre-v3.0.0 lwAFTR. + 3.0.1 + lwAFTR versions where "container" nodes in schemas are missing + corresponding nodes in the data unless "presence true" is + specified. + +The default version is "legacy". + +The resulting up-to-date configuration will be printed on standard output, ready to be saved to a new file. diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 2bcdc8af92..ad1fc0a576 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -24,10 +24,12 @@ end local function parse_args(args) local handlers = {} + local version = 'legacy' function handlers.h() show_usage(0) end - args = lib.dogetopt(args, handlers, "h", { help="h" }) + function handlers.f(v) version = string.lower(v) end + args = lib.dogetopt(args, handlers, "hf:", { help="h", from="f" }) if #args ~= 1 then show_usage(1) end - return unpack(args) + return args[1], version end local policies = { @@ -340,14 +342,26 @@ local function migrate_conf(old) } end -local function load_legacy_lwaftr_config(stream) +local function migrate_legacy(stream) local conf = Parser.new(stream):parse_property_list(lwaftr_conf_spec) return migrate_conf(conf) end +local function migrate_3_0_1(conf_file) + local data = require('lib.yang.data') + local str = "softwire-config {\n"..io.open(conf_file, 'r'):read('*a').."\n}" + return data.load_data_for_schema_by_name('snabb-softwire-v1', str, conf_file) +end + +local migrators = { legacy = migrate_legacy, ['3.0.1'] = migrate_3_0_1 } function run(args) - local conf_file = parse_args(args) - local conf = load_legacy_lwaftr_config(conf_file) + local conf_file, version = parse_args(args) + local migrate = migrators[version] + if not migrate then + io.stderr:write("error: unknown version: "..version.."\n") + show_usage(1) + end + local conf = migrate(conf_file) yang.print_data_for_schema_by_name('snabb-softwire-v1', conf, io.stdout) main.exit(0) end diff --git a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf index 2b72195ac0..f6cbceff2d 100644 --- a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf @@ -1,124 +1,126 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1960; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 2000; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1960; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 2000; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/icmp_on_fail.conf b/src/program/lwaftr/tests/data/icmp_on_fail.conf index 9490b763e4..ae41560560 100644 --- a/src/program/lwaftr/tests/data/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/icmp_on_fail.conf @@ -1,120 +1,122 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/no_hairpin.conf b/src/program/lwaftr/tests/data/no_hairpin.conf index 139ddd328c..d9df3aeac2 100644 --- a/src/program/lwaftr/tests/data/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/no_hairpin.conf @@ -1,123 +1,125 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - hairpinning false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + hairpinning false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/no_icmp.conf b/src/program/lwaftr/tests/data/no_icmp.conf index 72e5e0cf99..2b21e45d59 100644 --- a/src/program/lwaftr/tests/data/no_icmp.conf +++ b/src/program/lwaftr/tests/data/no_icmp.conf @@ -1,122 +1,124 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf index 63cc17d32d..b42ef58b8f 100644 --- a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf @@ -1,123 +1,125 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 1; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 1; - max-packets 10; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 1; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 1; + max-packets 10; + } } } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf index 9ab5faacee..d78af5683b 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf @@ -1,126 +1,128 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - egress-filter ip; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter ip; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - egress-filter ip6; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter ip6; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + egress-filter ip; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter ip; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + egress-filter ip6; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter ip6; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf index ea3d03ea09..d97d9e077f 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf @@ -1,128 +1,130 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + egress-filter ip; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter ip; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + egress-filter ip6; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter ip6; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - egress-filter ip; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter ip; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - egress-filter ip6; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter ip6; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf index 646e66db82..584afa9ea2 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf @@ -1,128 +1,130 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - egress-filter "len < 0"; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter "len < 0"; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - egress-filter "len < 0"; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter "len < 0"; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf index be1d7a02ca..2e63cadb82 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf @@ -1,126 +1,128 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - egress-filter "len < 0"; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter "len < 0"; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - egress-filter "len < 0"; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter "len < 0"; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf index 7b3ef4e296..f46104852c 100644 --- a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf @@ -1,119 +1,121 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 576; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 576; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf index 4676b2a84f..a7df23b0ee 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf @@ -1,124 +1,126 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1500; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 1280; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf index 1deadeba53..0e65ddd1c4 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf @@ -1,120 +1,122 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1500; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 1280; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf index 7ab70a365c..775259a935 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf @@ -1,122 +1,124 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1500; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 1280; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/tunnel_icmp.conf b/src/program/lwaftr/tests/data/tunnel_icmp.conf index 461e92d7e0..15b189b341 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp.conf @@ -1,118 +1,120 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf index c5a2dd5e3c..2751c2ce51 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf @@ -1,118 +1,120 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - ip 4.5.6.7; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + ip 4.5.6.7; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf index fb2d00fe97..0ce5c32a95 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf @@ -1,118 +1,120 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; - } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - ip 8:9:a:b:4:3:2:1; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + ip 8:9:a:b:4:3:2:1; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/lwaftr/tests/data/vlan.conf b/src/program/lwaftr/tests/data/vlan.conf index d6e9733278..f05fe81a61 100644 --- a/src/program/lwaftr/tests/data/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan.conf @@ -1,120 +1,122 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 6000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 6000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf index ec05e9087c..3ffd81afa5 100644 --- a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf @@ -1,126 +1,128 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1960; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 2000; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1960; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 2000; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf index ca44ef6d99..e89cfb6a04 100644 --- a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf @@ -1,122 +1,124 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf index 5eebdb78bc..1a31815ad9 100644 --- a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf @@ -1,125 +1,127 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + hairpinning false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - hairpinning false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp.conf b/src/program/lwaftr/tests/data/vlan/no_icmp.conf index 24469e9e73..ba864dd4c2 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp.conf @@ -1,124 +1,126 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf index d6629ac14f..6a2bd96a8a 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf @@ -1,125 +1,127 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 1; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 1; + max-packets 10; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 1; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 1; - max-packets 10; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf index ea3d03ea09..d97d9e077f 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf @@ -1,128 +1,130 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + egress-filter ip; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter ip; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + egress-filter ip6; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter ip6; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - egress-filter ip; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter ip; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - egress-filter ip6; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter ip6; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf index 646e66db82..584afa9ea2 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf @@ -1,128 +1,130 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + egress-filter "len < 0"; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ingress-filter "len < 0"; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - egress-filter "len < 0"; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter "len < 0"; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - egress-filter "len < 0"; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ingress-filter "len < 0"; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf index 74feec9219..54fdfe5699 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf @@ -1,121 +1,123 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 576; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 576; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf index c54940c80d..f42826c924 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf @@ -1,126 +1,128 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1500; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 1280; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf index 7ab70a365c..775259a935 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf @@ -1,122 +1,124 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 1500; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 1280; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - mtu 1500; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - mtu 1280; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf index e51c702c6e..a3850cb57f 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf @@ -1,120 +1,122 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf index a805111a9c..97def6e88b 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf @@ -1,120 +1,122 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + ip 4.5.6.7; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - ip 4.5.6.7; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf index 79f90d86ea..2b67e14617 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf @@ -1,120 +1,122 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + ip 8:9:a:b:4:3:2:1; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 600000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - ip 8:9:a:b:4:3:2:1; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/lwaftr/tests/data/vlan/vlan.conf b/src/program/lwaftr/tests/data/vlan/vlan.conf index d6e9733278..f05fe81a61 100644 --- a/src/program/lwaftr/tests/data/vlan/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan/vlan.conf @@ -1,120 +1,122 @@ -binding-table { - br-address 8:9:a:b:c:d:e:f; - br-address 1e:1:1:1:1:1:1:af; - br-address 1e:2:2:2:2:2:2:af; - psid-map { - addr 178.79.150.15; - psid-length 4; - shift 12; +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + psid-map { + addr 178.79.150.233; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.1; + psid-length 0; + shift 16; + } + psid-map { + addr 178.79.150.2; + psid-length 16; + shift 0; + } + psid-map { + addr 178.79.150.3; + psid-length 6; + shift 10; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.1; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.233; + psid 80; + b4-ipv6 127:2:3:4:5:6:7:128; + } + softwire { + ipv4 178.79.150.233; + psid 54192; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.3; + psid 4; + b4-ipv6 127:14:25:36:47:58:69:128; + br 2; + } + softwire { + ipv4 178.79.150.2; + psid 7850; + b4-ipv6 127:24:35:46:57:68:79:128; + br 1; + } + softwire { + ipv4 178.79.150.233; + psid 7850; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2300; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 22788; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.233; + psid 2700; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.15; + psid 1; + b4-ipv6 127:22:33:44:55:66:77:128; + } + softwire { + ipv4 178.79.150.15; + psid 0; + b4-ipv6 127:22:33:44:55:66:77:128; + } + } + external-interface { + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + error-rate-limiting { + packets 6000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - psid-map { - addr 178.79.150.233; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.1; - psid-length 0; - shift 16; - } - psid-map { - addr 178.79.150.2; - psid-length 16; - shift 0; - } - psid-map { - addr 178.79.150.3; - psid-length 6; - shift 10; - } - softwire { - ipv4 178.79.150.233; - psid 4660; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.1; - psid 0; - b4-ipv6 127:10:20:30:40:50:60:128; - } - softwire { - ipv4 178.79.150.233; - psid 80; - b4-ipv6 127:2:3:4:5:6:7:128; - } - softwire { - ipv4 178.79.150.233; - psid 54192; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.3; - psid 4; - b4-ipv6 127:14:25:36:47:58:69:128; - br 2; - } - softwire { - ipv4 178.79.150.2; - psid 7850; - b4-ipv6 127:24:35:46:57:68:79:128; - br 1; - } - softwire { - ipv4 178.79.150.233; - psid 7850; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2300; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 22788; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.233; - psid 2700; - b4-ipv6 127:11:12:13:14:15:16:128; - } - softwire { - ipv4 178.79.150.15; - psid 1; - b4-ipv6 127:22:33:44:55:66:77:128; - } - softwire { - ipv4 178.79.150.15; - psid 0; - b4-ipv6 127:22:33:44:55:66:77:128; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 10.10.10.10; - mac 12:12:12:12:12:12; - next-hop { - mac 68:68:68:68:68:68; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 6000; - } - ip 8:9:a:b:c:d:e:f; - mac 22:22:22:22:22:22; - next-hop { - mac 44:44:44:44:44:44; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf index 8f618f3f9f..e60687dfd3 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf @@ -1,985 +1,987 @@ -binding-table { - br-address fc00::100; - psid-map { - addr 193.5.1.100; - end-addr 193.5.1.102; - psid-length 6; - shift 10; - } - softwire { - ipv4 193.5.1.100; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:a8; - } - softwire { - ipv4 193.5.1.100; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:aa; - } - softwire { - ipv4 193.5.1.101; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:f8; - } - softwire { - ipv4 193.5.1.101; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:ce; - } - softwire { - ipv4 193.5.1.101; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:eb; - } - softwire { - ipv4 193.5.1.100; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:9e; - } - softwire { - ipv4 193.5.1.101; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:fb; - } - softwire { - ipv4 193.5.1.102; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:119; - } - softwire { - ipv4 193.5.1.101; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ed; - } - softwire { - ipv4 193.5.1.101; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:c5; - } - softwire { - ipv4 193.5.1.102; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:fe; - } - softwire { - ipv4 193.5.1.100; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:b5; - } - softwire { - ipv4 193.5.1.100; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:b0; - } - softwire { - ipv4 193.5.1.101; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:d6; - } - softwire { - ipv4 193.5.1.100; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:a2; - } - softwire { - ipv4 193.5.1.101; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:e8; - } - softwire { - ipv4 193.5.1.100; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:8e; - } - softwire { - ipv4 193.5.1.101; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ec; - } - softwire { - ipv4 193.5.1.102; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:122; - } - softwire { - ipv4 193.5.1.101; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:c1; - } - softwire { - ipv4 193.5.1.102; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:121; - } - softwire { - ipv4 193.5.1.101; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:d1; - } - softwire { - ipv4 193.5.1.101; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:ee; - } - softwire { - ipv4 193.5.1.101; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:df; - } - softwire { - ipv4 193.5.1.100; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:b4; - } - softwire { - ipv4 193.5.1.100; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ab; - } - softwire { - ipv4 193.5.1.100; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:b2; - } - softwire { - ipv4 193.5.1.100; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:b7; - } - softwire { - ipv4 193.5.1.101; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:f3; - } - softwire { - ipv4 193.5.1.101; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:bd; - } - softwire { - ipv4 193.5.1.100; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:ba; - } - softwire { - ipv4 193.5.1.100; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:9c; - } - softwire { - ipv4 193.5.1.102; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:116; - } - softwire { - ipv4 193.5.1.101; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:f2; - } - softwire { - ipv4 193.5.1.102; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:113; - } - softwire { - ipv4 193.5.1.100; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:88; - } - softwire { - ipv4 193.5.1.100; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:9f; - } - softwire { - ipv4 193.5.1.102; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:129; - } - softwire { - ipv4 193.5.1.101; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:cc; - } - softwire { - ipv4 193.5.1.102; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:10e; - } - softwire { - ipv4 193.5.1.102; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:13a; - } - softwire { - ipv4 193.5.1.100; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:8c; - } - softwire { - ipv4 193.5.1.100; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:8a; - } - softwire { - ipv4 193.5.1.102; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:123; - } - softwire { - ipv4 193.5.1.102; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:fd; - } - softwire { - ipv4 193.5.1.102; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:110; - } - softwire { - ipv4 193.5.1.101; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:cf; - } - softwire { - ipv4 193.5.1.102; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:138; - } - softwire { - ipv4 193.5.1.100; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:9d; - } - softwire { - ipv4 193.5.1.102; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:109; - } - softwire { - ipv4 193.5.1.100; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:96; - } - softwire { - ipv4 193.5.1.100; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:91; - } - softwire { - ipv4 193.5.1.100; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:b8; - } - softwire { - ipv4 193.5.1.102; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:104; - } - softwire { - ipv4 193.5.1.101; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:d3; - } - softwire { - ipv4 193.5.1.102; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:137; - } - softwire { - ipv4 193.5.1.101; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:e7; - } - softwire { - ipv4 193.5.1.102; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:108; - } - softwire { - ipv4 193.5.1.101; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:dd; - } - softwire { - ipv4 193.5.1.102; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:12a; - } - softwire { - ipv4 193.5.1.102; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:120; - } - softwire { - ipv4 193.5.1.100; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:83; - } - softwire { - ipv4 193.5.1.100; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ad; - } - softwire { - ipv4 193.5.1.100; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:9b; - } - softwire { - ipv4 193.5.1.100; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:bb; - } - softwire { - ipv4 193.5.1.102; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:10b; - } - softwire { - ipv4 193.5.1.100; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:af; - } - softwire { - ipv4 193.5.1.101; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:e6; - } - softwire { - ipv4 193.5.1.102; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:118; - } - softwire { - ipv4 193.5.1.100; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:b1; - } - softwire { - ipv4 193.5.1.102; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:133; - } - softwire { - ipv4 193.5.1.100; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ae; - } - softwire { - ipv4 193.5.1.101; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:f1; - } - softwire { - ipv4 193.5.1.102; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:134; - } - softwire { - ipv4 193.5.1.100; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:a1; - } - softwire { - ipv4 193.5.1.102; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:100; - } - softwire { - ipv4 193.5.1.101; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:db; - } - softwire { - ipv4 193.5.1.102; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:135; - } - softwire { - ipv4 193.5.1.102; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:11e; - } - softwire { - ipv4 193.5.1.100; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:a7; - } - softwire { - ipv4 193.5.1.102; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:ff; - } - softwire { - ipv4 193.5.1.100; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:89; - } - softwire { - ipv4 193.5.1.100; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:80; - } - softwire { - ipv4 193.5.1.100; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:84; - } - softwire { - ipv4 193.5.1.102; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:126; - } - softwire { - ipv4 193.5.1.101; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:c6; - } - softwire { - ipv4 193.5.1.100; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:b9; - } - softwire { - ipv4 193.5.1.101; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:d2; - } - softwire { - ipv4 193.5.1.101; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:dc; - } - softwire { - ipv4 193.5.1.102; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:112; - } - softwire { - ipv4 193.5.1.102; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:124; - } - softwire { - ipv4 193.5.1.100; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:90; - } - softwire { - ipv4 193.5.1.101; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:e1; - } - softwire { - ipv4 193.5.1.101; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:d5; - } - softwire { - ipv4 193.5.1.100; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:a3; - } - softwire { - ipv4 193.5.1.100; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:94; - } - softwire { - ipv4 193.5.1.100; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:a5; - } - softwire { - ipv4 193.5.1.100; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:7e; - } - softwire { - ipv4 193.5.1.102; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:132; - } - softwire { - ipv4 193.5.1.101; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:e4; - } - softwire { - ipv4 193.5.1.102; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:10a; - } - softwire { - ipv4 193.5.1.101; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:c2; - } - softwire { - ipv4 193.5.1.100; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:99; - } - softwire { - ipv4 193.5.1.101; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:e5; - } - softwire { - ipv4 193.5.1.101; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:c3; - } - softwire { - ipv4 193.5.1.101; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:fa; - } - softwire { - ipv4 193.5.1.102; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:128; - } - softwire { - ipv4 193.5.1.102; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:11a; - } - softwire { - ipv4 193.5.1.101; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:c9; - } - softwire { - ipv4 193.5.1.101; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:f7; - } - softwire { - ipv4 193.5.1.101; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:f5; - } - softwire { - ipv4 193.5.1.102; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:106; - } - softwire { - ipv4 193.5.1.101; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ea; - } - softwire { - ipv4 193.5.1.102; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:11d; - } - softwire { - ipv4 193.5.1.100; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:7f; - } - softwire { - ipv4 193.5.1.101; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:da; - } - softwire { - ipv4 193.5.1.101; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:e9; - } - softwire { - ipv4 193.5.1.101; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:f9; - } - softwire { - ipv4 193.5.1.102; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:10c; - } - softwire { - ipv4 193.5.1.101; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:d9; - } - softwire { - ipv4 193.5.1.102; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:11c; - } - softwire { - ipv4 193.5.1.101; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:c8; - } - softwire { - ipv4 193.5.1.101; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:c0; - } - softwire { - ipv4 193.5.1.101; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:f0; - } - softwire { - ipv4 193.5.1.102; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:10d; - } - softwire { - ipv4 193.5.1.101; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:bf; - } - softwire { - ipv4 193.5.1.102; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:105; - } - softwire { - ipv4 193.5.1.100; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:85; - } - softwire { - ipv4 193.5.1.101; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:be; - } - softwire { - ipv4 193.5.1.100; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:a4; - } - softwire { - ipv4 193.5.1.101; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:e0; - } - softwire { - ipv4 193.5.1.102; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:11b; - } - softwire { - ipv4 193.5.1.102; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:127; - } - softwire { - ipv4 193.5.1.102; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:12b; - } - softwire { - ipv4 193.5.1.102; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:10f; - } - softwire { - ipv4 193.5.1.100; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:8b; - } - softwire { - ipv4 193.5.1.100; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:95; - } - softwire { - ipv4 193.5.1.102; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:12e; - } - softwire { - ipv4 193.5.1.102; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:114; - } - softwire { - ipv4 193.5.1.100; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:87; - } - softwire { - ipv4 193.5.1.100; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:93; - } - softwire { - ipv4 193.5.1.102; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:103; - } - softwire { - ipv4 193.5.1.102; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:111; - } - softwire { - ipv4 193.5.1.101; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:f4; - } - softwire { - ipv4 193.5.1.100; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:98; - } - softwire { - ipv4 193.5.1.100; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:bc; - } - softwire { - ipv4 193.5.1.102; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:115; - } - softwire { - ipv4 193.5.1.100; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:b3; - } - softwire { - ipv4 193.5.1.100; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:a9; - } - softwire { - ipv4 193.5.1.102; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:12d; - } - softwire { - ipv4 193.5.1.100; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:86; - } - softwire { - ipv4 193.5.1.101; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:f6; - } - softwire { - ipv4 193.5.1.100; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:8f; - } - softwire { - ipv4 193.5.1.101; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:d8; - } - softwire { - ipv4 193.5.1.101; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:d7; - } - softwire { - ipv4 193.5.1.102; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:131; - } - softwire { - ipv4 193.5.1.102; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:136; - } - softwire { - ipv4 193.5.1.100; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:a6; - } - softwire { - ipv4 193.5.1.102; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:107; - } - softwire { - ipv4 193.5.1.100; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:a0; - } - softwire { - ipv4 193.5.1.101; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:c7; - } - softwire { - ipv4 193.5.1.100; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:ac; - } - softwire { - ipv4 193.5.1.101; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:d4; - } - softwire { - ipv4 193.5.1.102; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:fc; - } - softwire { - ipv4 193.5.1.101; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:c4; - } - softwire { - ipv4 193.5.1.101; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:ca; - } - softwire { - ipv4 193.5.1.101; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:cd; - } - softwire { - ipv4 193.5.1.102; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:102; - } - softwire { - ipv4 193.5.1.101; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:ef; - } - softwire { - ipv4 193.5.1.102; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:101; - } - softwire { - ipv4 193.5.1.102; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:139; - } - softwire { - ipv4 193.5.1.102; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:117; - } - softwire { - ipv4 193.5.1.100; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:8d; - } - softwire { - ipv4 193.5.1.100; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:9a; - } - softwire { - ipv4 193.5.1.102; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:125; - } - softwire { - ipv4 193.5.1.100; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:92; - } - softwire { - ipv4 193.5.1.102; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:12c; - } - softwire { - ipv4 193.5.1.100; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:b6; - } - softwire { - ipv4 193.5.1.101; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:d0; - } - softwire { - ipv4 193.5.1.102; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:12f; - } - softwire { - ipv4 193.5.1.100; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:97; - } - softwire { - ipv4 193.5.1.102; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:11f; - } - softwire { - ipv4 193.5.1.100; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:81; - } - softwire { - ipv4 193.5.1.101; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:de; - } - softwire { - ipv4 193.5.1.100; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:82; - } - softwire { - ipv4 193.5.1.101; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:e2; - } - softwire { - ipv4 193.5.1.101; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:cb; - } - softwire { - ipv4 193.5.1.101; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:e3; - } - softwire { - ipv4 193.5.1.102; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:130; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.0.1.1; - mac 02:aa:aa:aa:aa:aa; - next-hop { - mac 02:99:99:99:99:99; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip fc00::100; - mac 02:aa:aa:aa:aa:aa; - mtu 9500; - next-hop { - mac 02:99:99:99:99:99; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address fc00::100; + psid-map { + addr 193.5.1.100; + end-addr 193.5.1.102; + psid-length 6; + shift 10; + } + softwire { + ipv4 193.5.1.100; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:a8; + } + softwire { + ipv4 193.5.1.100; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:aa; + } + softwire { + ipv4 193.5.1.101; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:f8; + } + softwire { + ipv4 193.5.1.101; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:ce; + } + softwire { + ipv4 193.5.1.101; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:eb; + } + softwire { + ipv4 193.5.1.100; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:9e; + } + softwire { + ipv4 193.5.1.101; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:fb; + } + softwire { + ipv4 193.5.1.102; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:119; + } + softwire { + ipv4 193.5.1.101; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ed; + } + softwire { + ipv4 193.5.1.101; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:c5; + } + softwire { + ipv4 193.5.1.102; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:fe; + } + softwire { + ipv4 193.5.1.100; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:b5; + } + softwire { + ipv4 193.5.1.100; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:b0; + } + softwire { + ipv4 193.5.1.101; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:d6; + } + softwire { + ipv4 193.5.1.100; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:a2; + } + softwire { + ipv4 193.5.1.101; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:e8; + } + softwire { + ipv4 193.5.1.100; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:8e; + } + softwire { + ipv4 193.5.1.101; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ec; + } + softwire { + ipv4 193.5.1.102; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:122; + } + softwire { + ipv4 193.5.1.101; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:c1; + } + softwire { + ipv4 193.5.1.102; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:121; + } + softwire { + ipv4 193.5.1.101; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:d1; + } + softwire { + ipv4 193.5.1.101; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:ee; + } + softwire { + ipv4 193.5.1.101; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:df; + } + softwire { + ipv4 193.5.1.100; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:b4; + } + softwire { + ipv4 193.5.1.100; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ab; + } + softwire { + ipv4 193.5.1.100; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:b2; + } + softwire { + ipv4 193.5.1.100; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:b7; + } + softwire { + ipv4 193.5.1.101; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:f3; + } + softwire { + ipv4 193.5.1.101; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:bd; + } + softwire { + ipv4 193.5.1.100; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:ba; + } + softwire { + ipv4 193.5.1.100; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:9c; + } + softwire { + ipv4 193.5.1.102; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:116; + } + softwire { + ipv4 193.5.1.101; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:f2; + } + softwire { + ipv4 193.5.1.102; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:113; + } + softwire { + ipv4 193.5.1.100; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:88; + } + softwire { + ipv4 193.5.1.100; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:9f; + } + softwire { + ipv4 193.5.1.102; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:129; + } + softwire { + ipv4 193.5.1.101; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:cc; + } + softwire { + ipv4 193.5.1.102; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:10e; + } + softwire { + ipv4 193.5.1.102; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:13a; + } + softwire { + ipv4 193.5.1.100; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:8c; + } + softwire { + ipv4 193.5.1.100; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:8a; + } + softwire { + ipv4 193.5.1.102; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:123; + } + softwire { + ipv4 193.5.1.102; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:fd; + } + softwire { + ipv4 193.5.1.102; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:110; + } + softwire { + ipv4 193.5.1.101; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:cf; + } + softwire { + ipv4 193.5.1.102; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:138; + } + softwire { + ipv4 193.5.1.100; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:9d; + } + softwire { + ipv4 193.5.1.102; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:109; + } + softwire { + ipv4 193.5.1.100; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:96; + } + softwire { + ipv4 193.5.1.100; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:91; + } + softwire { + ipv4 193.5.1.100; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:b8; + } + softwire { + ipv4 193.5.1.102; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:104; + } + softwire { + ipv4 193.5.1.101; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:d3; + } + softwire { + ipv4 193.5.1.102; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:137; + } + softwire { + ipv4 193.5.1.101; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:e7; + } + softwire { + ipv4 193.5.1.102; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:108; + } + softwire { + ipv4 193.5.1.101; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:dd; + } + softwire { + ipv4 193.5.1.102; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:12a; + } + softwire { + ipv4 193.5.1.102; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:120; + } + softwire { + ipv4 193.5.1.100; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:83; + } + softwire { + ipv4 193.5.1.100; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ad; + } + softwire { + ipv4 193.5.1.100; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:9b; + } + softwire { + ipv4 193.5.1.100; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:bb; + } + softwire { + ipv4 193.5.1.102; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:10b; + } + softwire { + ipv4 193.5.1.100; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:af; + } + softwire { + ipv4 193.5.1.101; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:e6; + } + softwire { + ipv4 193.5.1.102; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:118; + } + softwire { + ipv4 193.5.1.100; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:b1; + } + softwire { + ipv4 193.5.1.102; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:133; + } + softwire { + ipv4 193.5.1.100; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ae; + } + softwire { + ipv4 193.5.1.101; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:f1; + } + softwire { + ipv4 193.5.1.102; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:134; + } + softwire { + ipv4 193.5.1.100; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:a1; + } + softwire { + ipv4 193.5.1.102; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:100; + } + softwire { + ipv4 193.5.1.101; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:db; + } + softwire { + ipv4 193.5.1.102; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:135; + } + softwire { + ipv4 193.5.1.102; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:11e; + } + softwire { + ipv4 193.5.1.100; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:a7; + } + softwire { + ipv4 193.5.1.102; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:ff; + } + softwire { + ipv4 193.5.1.100; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:89; + } + softwire { + ipv4 193.5.1.100; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:80; + } + softwire { + ipv4 193.5.1.100; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:84; + } + softwire { + ipv4 193.5.1.102; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:126; + } + softwire { + ipv4 193.5.1.101; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:c6; + } + softwire { + ipv4 193.5.1.100; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:b9; + } + softwire { + ipv4 193.5.1.101; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:d2; + } + softwire { + ipv4 193.5.1.101; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:dc; + } + softwire { + ipv4 193.5.1.102; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:112; + } + softwire { + ipv4 193.5.1.102; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:124; + } + softwire { + ipv4 193.5.1.100; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:90; + } + softwire { + ipv4 193.5.1.101; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:e1; + } + softwire { + ipv4 193.5.1.101; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:d5; + } + softwire { + ipv4 193.5.1.100; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:a3; + } + softwire { + ipv4 193.5.1.100; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:94; + } + softwire { + ipv4 193.5.1.100; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:a5; + } + softwire { + ipv4 193.5.1.100; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:7e; + } + softwire { + ipv4 193.5.1.102; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:132; + } + softwire { + ipv4 193.5.1.101; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:e4; + } + softwire { + ipv4 193.5.1.102; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:10a; + } + softwire { + ipv4 193.5.1.101; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:c2; + } + softwire { + ipv4 193.5.1.100; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:99; + } + softwire { + ipv4 193.5.1.101; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:e5; + } + softwire { + ipv4 193.5.1.101; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:c3; + } + softwire { + ipv4 193.5.1.101; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:fa; + } + softwire { + ipv4 193.5.1.102; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:128; + } + softwire { + ipv4 193.5.1.102; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:11a; + } + softwire { + ipv4 193.5.1.101; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:c9; + } + softwire { + ipv4 193.5.1.101; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:f7; + } + softwire { + ipv4 193.5.1.101; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:f5; + } + softwire { + ipv4 193.5.1.102; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:106; + } + softwire { + ipv4 193.5.1.101; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ea; + } + softwire { + ipv4 193.5.1.102; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:11d; + } + softwire { + ipv4 193.5.1.100; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:7f; + } + softwire { + ipv4 193.5.1.101; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:da; + } + softwire { + ipv4 193.5.1.101; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:e9; + } + softwire { + ipv4 193.5.1.101; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:f9; + } + softwire { + ipv4 193.5.1.102; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:10c; + } + softwire { + ipv4 193.5.1.101; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:d9; + } + softwire { + ipv4 193.5.1.102; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:11c; + } + softwire { + ipv4 193.5.1.101; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:c8; + } + softwire { + ipv4 193.5.1.101; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:c0; + } + softwire { + ipv4 193.5.1.101; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:f0; + } + softwire { + ipv4 193.5.1.102; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:10d; + } + softwire { + ipv4 193.5.1.101; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:bf; + } + softwire { + ipv4 193.5.1.102; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:105; + } + softwire { + ipv4 193.5.1.100; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:85; + } + softwire { + ipv4 193.5.1.101; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:be; + } + softwire { + ipv4 193.5.1.100; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:a4; + } + softwire { + ipv4 193.5.1.101; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:e0; + } + softwire { + ipv4 193.5.1.102; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:11b; + } + softwire { + ipv4 193.5.1.102; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:127; + } + softwire { + ipv4 193.5.1.102; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:12b; + } + softwire { + ipv4 193.5.1.102; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:10f; + } + softwire { + ipv4 193.5.1.100; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:8b; + } + softwire { + ipv4 193.5.1.100; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:95; + } + softwire { + ipv4 193.5.1.102; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:12e; + } + softwire { + ipv4 193.5.1.102; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:114; + } + softwire { + ipv4 193.5.1.100; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:87; + } + softwire { + ipv4 193.5.1.100; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:93; + } + softwire { + ipv4 193.5.1.102; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:103; + } + softwire { + ipv4 193.5.1.102; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:111; + } + softwire { + ipv4 193.5.1.101; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:f4; + } + softwire { + ipv4 193.5.1.100; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:98; + } + softwire { + ipv4 193.5.1.100; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:bc; + } + softwire { + ipv4 193.5.1.102; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:115; + } + softwire { + ipv4 193.5.1.100; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:b3; + } + softwire { + ipv4 193.5.1.100; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:a9; + } + softwire { + ipv4 193.5.1.102; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:12d; + } + softwire { + ipv4 193.5.1.100; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:86; + } + softwire { + ipv4 193.5.1.101; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:f6; + } + softwire { + ipv4 193.5.1.100; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:8f; + } + softwire { + ipv4 193.5.1.101; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:d8; + } + softwire { + ipv4 193.5.1.101; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:d7; + } + softwire { + ipv4 193.5.1.102; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:131; + } + softwire { + ipv4 193.5.1.102; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:136; + } + softwire { + ipv4 193.5.1.100; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:a6; + } + softwire { + ipv4 193.5.1.102; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:107; + } + softwire { + ipv4 193.5.1.100; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:a0; + } + softwire { + ipv4 193.5.1.101; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:c7; + } + softwire { + ipv4 193.5.1.100; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:ac; + } + softwire { + ipv4 193.5.1.101; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:d4; + } + softwire { + ipv4 193.5.1.102; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:fc; + } + softwire { + ipv4 193.5.1.101; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:c4; + } + softwire { + ipv4 193.5.1.101; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:ca; + } + softwire { + ipv4 193.5.1.101; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:cd; + } + softwire { + ipv4 193.5.1.102; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:102; + } + softwire { + ipv4 193.5.1.101; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:ef; + } + softwire { + ipv4 193.5.1.102; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:101; + } + softwire { + ipv4 193.5.1.102; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:139; + } + softwire { + ipv4 193.5.1.102; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:117; + } + softwire { + ipv4 193.5.1.100; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:8d; + } + softwire { + ipv4 193.5.1.100; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:9a; + } + softwire { + ipv4 193.5.1.102; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:125; + } + softwire { + ipv4 193.5.1.100; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:92; + } + softwire { + ipv4 193.5.1.102; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:12c; + } + softwire { + ipv4 193.5.1.100; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:b6; + } + softwire { + ipv4 193.5.1.101; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:d0; + } + softwire { + ipv4 193.5.1.102; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:12f; + } + softwire { + ipv4 193.5.1.100; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:97; + } + softwire { + ipv4 193.5.1.102; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:11f; + } + softwire { + ipv4 193.5.1.100; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:81; + } + softwire { + ipv4 193.5.1.101; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:de; + } + softwire { + ipv4 193.5.1.100; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:82; + } + softwire { + ipv4 193.5.1.101; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:e2; + } + softwire { + ipv4 193.5.1.101; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:cb; + } + softwire { + ipv4 193.5.1.101; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:e3; + } + softwire { + ipv4 193.5.1.102; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:130; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.0.1.1; + mac 02:aa:aa:aa:aa:aa; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip fc00::100; + mac 02:aa:aa:aa:aa:aa; + mtu 9500; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf index 8f618f3f9f..e60687dfd3 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf @@ -1,985 +1,987 @@ -binding-table { - br-address fc00::100; - psid-map { - addr 193.5.1.100; - end-addr 193.5.1.102; - psid-length 6; - shift 10; - } - softwire { - ipv4 193.5.1.100; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:a8; - } - softwire { - ipv4 193.5.1.100; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:aa; - } - softwire { - ipv4 193.5.1.101; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:f8; - } - softwire { - ipv4 193.5.1.101; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:ce; - } - softwire { - ipv4 193.5.1.101; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:eb; - } - softwire { - ipv4 193.5.1.100; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:9e; - } - softwire { - ipv4 193.5.1.101; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:fb; - } - softwire { - ipv4 193.5.1.102; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:119; - } - softwire { - ipv4 193.5.1.101; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ed; - } - softwire { - ipv4 193.5.1.101; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:c5; - } - softwire { - ipv4 193.5.1.102; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:fe; - } - softwire { - ipv4 193.5.1.100; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:b5; - } - softwire { - ipv4 193.5.1.100; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:b0; - } - softwire { - ipv4 193.5.1.101; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:d6; - } - softwire { - ipv4 193.5.1.100; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:a2; - } - softwire { - ipv4 193.5.1.101; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:e8; - } - softwire { - ipv4 193.5.1.100; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:8e; - } - softwire { - ipv4 193.5.1.101; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ec; - } - softwire { - ipv4 193.5.1.102; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:122; - } - softwire { - ipv4 193.5.1.101; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:c1; - } - softwire { - ipv4 193.5.1.102; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:121; - } - softwire { - ipv4 193.5.1.101; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:d1; - } - softwire { - ipv4 193.5.1.101; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:ee; - } - softwire { - ipv4 193.5.1.101; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:df; - } - softwire { - ipv4 193.5.1.100; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:b4; - } - softwire { - ipv4 193.5.1.100; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ab; - } - softwire { - ipv4 193.5.1.100; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:b2; - } - softwire { - ipv4 193.5.1.100; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:b7; - } - softwire { - ipv4 193.5.1.101; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:f3; - } - softwire { - ipv4 193.5.1.101; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:bd; - } - softwire { - ipv4 193.5.1.100; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:ba; - } - softwire { - ipv4 193.5.1.100; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:9c; - } - softwire { - ipv4 193.5.1.102; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:116; - } - softwire { - ipv4 193.5.1.101; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:f2; - } - softwire { - ipv4 193.5.1.102; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:113; - } - softwire { - ipv4 193.5.1.100; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:88; - } - softwire { - ipv4 193.5.1.100; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:9f; - } - softwire { - ipv4 193.5.1.102; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:129; - } - softwire { - ipv4 193.5.1.101; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:cc; - } - softwire { - ipv4 193.5.1.102; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:10e; - } - softwire { - ipv4 193.5.1.102; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:13a; - } - softwire { - ipv4 193.5.1.100; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:8c; - } - softwire { - ipv4 193.5.1.100; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:8a; - } - softwire { - ipv4 193.5.1.102; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:123; - } - softwire { - ipv4 193.5.1.102; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:fd; - } - softwire { - ipv4 193.5.1.102; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:110; - } - softwire { - ipv4 193.5.1.101; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:cf; - } - softwire { - ipv4 193.5.1.102; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:138; - } - softwire { - ipv4 193.5.1.100; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:9d; - } - softwire { - ipv4 193.5.1.102; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:109; - } - softwire { - ipv4 193.5.1.100; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:96; - } - softwire { - ipv4 193.5.1.100; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:91; - } - softwire { - ipv4 193.5.1.100; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:b8; - } - softwire { - ipv4 193.5.1.102; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:104; - } - softwire { - ipv4 193.5.1.101; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:d3; - } - softwire { - ipv4 193.5.1.102; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:137; - } - softwire { - ipv4 193.5.1.101; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:e7; - } - softwire { - ipv4 193.5.1.102; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:108; - } - softwire { - ipv4 193.5.1.101; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:dd; - } - softwire { - ipv4 193.5.1.102; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:12a; - } - softwire { - ipv4 193.5.1.102; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:120; - } - softwire { - ipv4 193.5.1.100; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:83; - } - softwire { - ipv4 193.5.1.100; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:ad; - } - softwire { - ipv4 193.5.1.100; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:9b; - } - softwire { - ipv4 193.5.1.100; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:bb; - } - softwire { - ipv4 193.5.1.102; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:10b; - } - softwire { - ipv4 193.5.1.100; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:af; - } - softwire { - ipv4 193.5.1.101; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:e6; - } - softwire { - ipv4 193.5.1.102; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:118; - } - softwire { - ipv4 193.5.1.100; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:b1; - } - softwire { - ipv4 193.5.1.102; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:133; - } - softwire { - ipv4 193.5.1.100; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:ae; - } - softwire { - ipv4 193.5.1.101; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:f1; - } - softwire { - ipv4 193.5.1.102; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:134; - } - softwire { - ipv4 193.5.1.100; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:a1; - } - softwire { - ipv4 193.5.1.102; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:100; - } - softwire { - ipv4 193.5.1.101; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:db; - } - softwire { - ipv4 193.5.1.102; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:135; - } - softwire { - ipv4 193.5.1.102; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:11e; - } - softwire { - ipv4 193.5.1.100; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:a7; - } - softwire { - ipv4 193.5.1.102; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:ff; - } - softwire { - ipv4 193.5.1.100; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:89; - } - softwire { - ipv4 193.5.1.100; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:80; - } - softwire { - ipv4 193.5.1.100; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:84; - } - softwire { - ipv4 193.5.1.102; - psid 43; - b4-ipv6 fc00:1:2:3:4:5:0:126; - } - softwire { - ipv4 193.5.1.101; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:c6; - } - softwire { - ipv4 193.5.1.100; - psid 60; - b4-ipv6 fc00:1:2:3:4:5:0:b9; - } - softwire { - ipv4 193.5.1.101; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:d2; - } - softwire { - ipv4 193.5.1.101; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:dc; - } - softwire { - ipv4 193.5.1.102; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:112; - } - softwire { - ipv4 193.5.1.102; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:124; - } - softwire { - ipv4 193.5.1.100; - psid 19; - b4-ipv6 fc00:1:2:3:4:5:0:90; - } - softwire { - ipv4 193.5.1.101; - psid 37; - b4-ipv6 fc00:1:2:3:4:5:0:e1; - } - softwire { - ipv4 193.5.1.101; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:d5; - } - softwire { - ipv4 193.5.1.100; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:a3; - } - softwire { - ipv4 193.5.1.100; - psid 23; - b4-ipv6 fc00:1:2:3:4:5:0:94; - } - softwire { - ipv4 193.5.1.100; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:a5; - } - softwire { - ipv4 193.5.1.100; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:7e; - } - softwire { - ipv4 193.5.1.102; - psid 55; - b4-ipv6 fc00:1:2:3:4:5:0:132; - } - softwire { - ipv4 193.5.1.101; - psid 40; - b4-ipv6 fc00:1:2:3:4:5:0:e4; - } - softwire { - ipv4 193.5.1.102; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:10a; - } - softwire { - ipv4 193.5.1.101; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:c2; - } - softwire { - ipv4 193.5.1.100; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:99; - } - softwire { - ipv4 193.5.1.101; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:e5; - } - softwire { - ipv4 193.5.1.101; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:c3; - } - softwire { - ipv4 193.5.1.101; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:fa; - } - softwire { - ipv4 193.5.1.102; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:128; - } - softwire { - ipv4 193.5.1.102; - psid 31; - b4-ipv6 fc00:1:2:3:4:5:0:11a; - } - softwire { - ipv4 193.5.1.101; - psid 13; - b4-ipv6 fc00:1:2:3:4:5:0:c9; - } - softwire { - ipv4 193.5.1.101; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:f7; - } - softwire { - ipv4 193.5.1.101; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:f5; - } - softwire { - ipv4 193.5.1.102; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:106; - } - softwire { - ipv4 193.5.1.101; - psid 46; - b4-ipv6 fc00:1:2:3:4:5:0:ea; - } - softwire { - ipv4 193.5.1.102; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:11d; - } - softwire { - ipv4 193.5.1.100; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:7f; - } - softwire { - ipv4 193.5.1.101; - psid 30; - b4-ipv6 fc00:1:2:3:4:5:0:da; - } - softwire { - ipv4 193.5.1.101; - psid 45; - b4-ipv6 fc00:1:2:3:4:5:0:e9; - } - softwire { - ipv4 193.5.1.101; - psid 61; - b4-ipv6 fc00:1:2:3:4:5:0:f9; - } - softwire { - ipv4 193.5.1.102; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:10c; - } - softwire { - ipv4 193.5.1.101; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:d9; - } - softwire { - ipv4 193.5.1.102; - psid 33; - b4-ipv6 fc00:1:2:3:4:5:0:11c; - } - softwire { - ipv4 193.5.1.101; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:c8; - } - softwire { - ipv4 193.5.1.101; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:c0; - } - softwire { - ipv4 193.5.1.101; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:f0; - } - softwire { - ipv4 193.5.1.102; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:10d; - } - softwire { - ipv4 193.5.1.101; - psid 3; - b4-ipv6 fc00:1:2:3:4:5:0:bf; - } - softwire { - ipv4 193.5.1.102; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:105; - } - softwire { - ipv4 193.5.1.100; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:85; - } - softwire { - ipv4 193.5.1.101; - psid 2; - b4-ipv6 fc00:1:2:3:4:5:0:be; - } - softwire { - ipv4 193.5.1.100; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:a4; - } - softwire { - ipv4 193.5.1.101; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:e0; - } - softwire { - ipv4 193.5.1.102; - psid 32; - b4-ipv6 fc00:1:2:3:4:5:0:11b; - } - softwire { - ipv4 193.5.1.102; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:127; - } - softwire { - ipv4 193.5.1.102; - psid 48; - b4-ipv6 fc00:1:2:3:4:5:0:12b; - } - softwire { - ipv4 193.5.1.102; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:10f; - } - softwire { - ipv4 193.5.1.100; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:8b; - } - softwire { - ipv4 193.5.1.100; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:95; - } - softwire { - ipv4 193.5.1.102; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:12e; - } - softwire { - ipv4 193.5.1.102; - psid 25; - b4-ipv6 fc00:1:2:3:4:5:0:114; - } - softwire { - ipv4 193.5.1.100; - psid 10; - b4-ipv6 fc00:1:2:3:4:5:0:87; - } - softwire { - ipv4 193.5.1.100; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:93; - } - softwire { - ipv4 193.5.1.102; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:103; - } - softwire { - ipv4 193.5.1.102; - psid 22; - b4-ipv6 fc00:1:2:3:4:5:0:111; - } - softwire { - ipv4 193.5.1.101; - psid 56; - b4-ipv6 fc00:1:2:3:4:5:0:f4; - } - softwire { - ipv4 193.5.1.100; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:98; - } - softwire { - ipv4 193.5.1.100; - psid 63; - b4-ipv6 fc00:1:2:3:4:5:0:bc; - } - softwire { - ipv4 193.5.1.102; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:115; - } - softwire { - ipv4 193.5.1.100; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:b3; - } - softwire { - ipv4 193.5.1.100; - psid 44; - b4-ipv6 fc00:1:2:3:4:5:0:a9; - } - softwire { - ipv4 193.5.1.102; - psid 50; - b4-ipv6 fc00:1:2:3:4:5:0:12d; - } - softwire { - ipv4 193.5.1.100; - psid 9; - b4-ipv6 fc00:1:2:3:4:5:0:86; - } - softwire { - ipv4 193.5.1.101; - psid 58; - b4-ipv6 fc00:1:2:3:4:5:0:f6; - } - softwire { - ipv4 193.5.1.100; - psid 18; - b4-ipv6 fc00:1:2:3:4:5:0:8f; - } - softwire { - ipv4 193.5.1.101; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:d8; - } - softwire { - ipv4 193.5.1.101; - psid 27; - b4-ipv6 fc00:1:2:3:4:5:0:d7; - } - softwire { - ipv4 193.5.1.102; - psid 54; - b4-ipv6 fc00:1:2:3:4:5:0:131; - } - softwire { - ipv4 193.5.1.102; - psid 59; - b4-ipv6 fc00:1:2:3:4:5:0:136; - } - softwire { - ipv4 193.5.1.100; - psid 41; - b4-ipv6 fc00:1:2:3:4:5:0:a6; - } - softwire { - ipv4 193.5.1.102; - psid 12; - b4-ipv6 fc00:1:2:3:4:5:0:107; - } - softwire { - ipv4 193.5.1.100; - psid 35; - b4-ipv6 fc00:1:2:3:4:5:0:a0; - } - softwire { - ipv4 193.5.1.101; - psid 11; - b4-ipv6 fc00:1:2:3:4:5:0:c7; - } - softwire { - ipv4 193.5.1.100; - psid 47; - b4-ipv6 fc00:1:2:3:4:5:0:ac; - } - softwire { - ipv4 193.5.1.101; - psid 24; - b4-ipv6 fc00:1:2:3:4:5:0:d4; - } - softwire { - ipv4 193.5.1.102; - psid 1; - b4-ipv6 fc00:1:2:3:4:5:0:fc; - } - softwire { - ipv4 193.5.1.101; - psid 8; - b4-ipv6 fc00:1:2:3:4:5:0:c4; - } - softwire { - ipv4 193.5.1.101; - psid 14; - b4-ipv6 fc00:1:2:3:4:5:0:ca; - } - softwire { - ipv4 193.5.1.101; - psid 17; - b4-ipv6 fc00:1:2:3:4:5:0:cd; - } - softwire { - ipv4 193.5.1.102; - psid 7; - b4-ipv6 fc00:1:2:3:4:5:0:102; - } - softwire { - ipv4 193.5.1.101; - psid 51; - b4-ipv6 fc00:1:2:3:4:5:0:ef; - } - softwire { - ipv4 193.5.1.102; - psid 6; - b4-ipv6 fc00:1:2:3:4:5:0:101; - } - softwire { - ipv4 193.5.1.102; - psid 62; - b4-ipv6 fc00:1:2:3:4:5:0:139; - } - softwire { - ipv4 193.5.1.102; - psid 28; - b4-ipv6 fc00:1:2:3:4:5:0:117; - } - softwire { - ipv4 193.5.1.100; - psid 16; - b4-ipv6 fc00:1:2:3:4:5:0:8d; - } - softwire { - ipv4 193.5.1.100; - psid 29; - b4-ipv6 fc00:1:2:3:4:5:0:9a; - } - softwire { - ipv4 193.5.1.102; - psid 42; - b4-ipv6 fc00:1:2:3:4:5:0:125; - } - softwire { - ipv4 193.5.1.100; - psid 21; - b4-ipv6 fc00:1:2:3:4:5:0:92; - } - softwire { - ipv4 193.5.1.102; - psid 49; - b4-ipv6 fc00:1:2:3:4:5:0:12c; - } - softwire { - ipv4 193.5.1.100; - psid 57; - b4-ipv6 fc00:1:2:3:4:5:0:b6; - } - softwire { - ipv4 193.5.1.101; - psid 20; - b4-ipv6 fc00:1:2:3:4:5:0:d0; - } - softwire { - ipv4 193.5.1.102; - psid 52; - b4-ipv6 fc00:1:2:3:4:5:0:12f; - } - softwire { - ipv4 193.5.1.100; - psid 26; - b4-ipv6 fc00:1:2:3:4:5:0:97; - } - softwire { - ipv4 193.5.1.102; - psid 36; - b4-ipv6 fc00:1:2:3:4:5:0:11f; - } - softwire { - ipv4 193.5.1.100; - psid 4; - b4-ipv6 fc00:1:2:3:4:5:0:81; - } - softwire { - ipv4 193.5.1.101; - psid 34; - b4-ipv6 fc00:1:2:3:4:5:0:de; - } - softwire { - ipv4 193.5.1.100; - psid 5; - b4-ipv6 fc00:1:2:3:4:5:0:82; - } - softwire { - ipv4 193.5.1.101; - psid 38; - b4-ipv6 fc00:1:2:3:4:5:0:e2; - } - softwire { - ipv4 193.5.1.101; - psid 15; - b4-ipv6 fc00:1:2:3:4:5:0:cb; - } - softwire { - ipv4 193.5.1.101; - psid 39; - b4-ipv6 fc00:1:2:3:4:5:0:e3; - } - softwire { - ipv4 193.5.1.102; - psid 53; - b4-ipv6 fc00:1:2:3:4:5:0:130; - } -} -external-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip 10.0.1.1; - mac 02:aa:aa:aa:aa:aa; - next-hop { - mac 02:99:99:99:99:99; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - allow-incoming-icmp false; - error-rate-limiting { - packets 600000; - } - generate-icmp-errors false; - ip fc00::100; - mac 02:aa:aa:aa:aa:aa; - mtu 9500; - next-hop { - mac 02:99:99:99:99:99; - } - reassembly { - max-fragments-per-packet 40; +softwire-config { + binding-table { + br-address fc00::100; + psid-map { + addr 193.5.1.100; + end-addr 193.5.1.102; + psid-length 6; + shift 10; + } + softwire { + ipv4 193.5.1.100; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:a8; + } + softwire { + ipv4 193.5.1.100; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:aa; + } + softwire { + ipv4 193.5.1.101; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:f8; + } + softwire { + ipv4 193.5.1.101; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:ce; + } + softwire { + ipv4 193.5.1.101; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:eb; + } + softwire { + ipv4 193.5.1.100; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:9e; + } + softwire { + ipv4 193.5.1.101; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:fb; + } + softwire { + ipv4 193.5.1.102; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:119; + } + softwire { + ipv4 193.5.1.101; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ed; + } + softwire { + ipv4 193.5.1.101; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:c5; + } + softwire { + ipv4 193.5.1.102; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:fe; + } + softwire { + ipv4 193.5.1.100; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:b5; + } + softwire { + ipv4 193.5.1.100; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:b0; + } + softwire { + ipv4 193.5.1.101; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:d6; + } + softwire { + ipv4 193.5.1.100; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:a2; + } + softwire { + ipv4 193.5.1.101; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:e8; + } + softwire { + ipv4 193.5.1.100; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:8e; + } + softwire { + ipv4 193.5.1.101; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ec; + } + softwire { + ipv4 193.5.1.102; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:122; + } + softwire { + ipv4 193.5.1.101; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:c1; + } + softwire { + ipv4 193.5.1.102; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:121; + } + softwire { + ipv4 193.5.1.101; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:d1; + } + softwire { + ipv4 193.5.1.101; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:ee; + } + softwire { + ipv4 193.5.1.101; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:df; + } + softwire { + ipv4 193.5.1.100; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:b4; + } + softwire { + ipv4 193.5.1.100; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ab; + } + softwire { + ipv4 193.5.1.100; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:b2; + } + softwire { + ipv4 193.5.1.100; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:b7; + } + softwire { + ipv4 193.5.1.101; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:f3; + } + softwire { + ipv4 193.5.1.101; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:bd; + } + softwire { + ipv4 193.5.1.100; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:ba; + } + softwire { + ipv4 193.5.1.100; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:9c; + } + softwire { + ipv4 193.5.1.102; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:116; + } + softwire { + ipv4 193.5.1.101; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:f2; + } + softwire { + ipv4 193.5.1.102; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:113; + } + softwire { + ipv4 193.5.1.100; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:88; + } + softwire { + ipv4 193.5.1.100; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:9f; + } + softwire { + ipv4 193.5.1.102; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:129; + } + softwire { + ipv4 193.5.1.101; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:cc; + } + softwire { + ipv4 193.5.1.102; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:10e; + } + softwire { + ipv4 193.5.1.102; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:13a; + } + softwire { + ipv4 193.5.1.100; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:8c; + } + softwire { + ipv4 193.5.1.100; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:8a; + } + softwire { + ipv4 193.5.1.102; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:123; + } + softwire { + ipv4 193.5.1.102; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:fd; + } + softwire { + ipv4 193.5.1.102; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:110; + } + softwire { + ipv4 193.5.1.101; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:cf; + } + softwire { + ipv4 193.5.1.102; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:138; + } + softwire { + ipv4 193.5.1.100; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:9d; + } + softwire { + ipv4 193.5.1.102; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:109; + } + softwire { + ipv4 193.5.1.100; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:96; + } + softwire { + ipv4 193.5.1.100; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:91; + } + softwire { + ipv4 193.5.1.100; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:b8; + } + softwire { + ipv4 193.5.1.102; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:104; + } + softwire { + ipv4 193.5.1.101; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:d3; + } + softwire { + ipv4 193.5.1.102; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:137; + } + softwire { + ipv4 193.5.1.101; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:e7; + } + softwire { + ipv4 193.5.1.102; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:108; + } + softwire { + ipv4 193.5.1.101; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:dd; + } + softwire { + ipv4 193.5.1.102; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:12a; + } + softwire { + ipv4 193.5.1.102; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:120; + } + softwire { + ipv4 193.5.1.100; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:83; + } + softwire { + ipv4 193.5.1.100; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:ad; + } + softwire { + ipv4 193.5.1.100; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:9b; + } + softwire { + ipv4 193.5.1.100; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:bb; + } + softwire { + ipv4 193.5.1.102; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:10b; + } + softwire { + ipv4 193.5.1.100; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:af; + } + softwire { + ipv4 193.5.1.101; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:e6; + } + softwire { + ipv4 193.5.1.102; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:118; + } + softwire { + ipv4 193.5.1.100; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:b1; + } + softwire { + ipv4 193.5.1.102; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:133; + } + softwire { + ipv4 193.5.1.100; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:ae; + } + softwire { + ipv4 193.5.1.101; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:f1; + } + softwire { + ipv4 193.5.1.102; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:134; + } + softwire { + ipv4 193.5.1.100; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:a1; + } + softwire { + ipv4 193.5.1.102; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:100; + } + softwire { + ipv4 193.5.1.101; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:db; + } + softwire { + ipv4 193.5.1.102; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:135; + } + softwire { + ipv4 193.5.1.102; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:11e; + } + softwire { + ipv4 193.5.1.100; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:a7; + } + softwire { + ipv4 193.5.1.102; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:ff; + } + softwire { + ipv4 193.5.1.100; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:89; + } + softwire { + ipv4 193.5.1.100; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:80; + } + softwire { + ipv4 193.5.1.100; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:84; + } + softwire { + ipv4 193.5.1.102; + psid 43; + b4-ipv6 fc00:1:2:3:4:5:0:126; + } + softwire { + ipv4 193.5.1.101; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:c6; + } + softwire { + ipv4 193.5.1.100; + psid 60; + b4-ipv6 fc00:1:2:3:4:5:0:b9; + } + softwire { + ipv4 193.5.1.101; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:d2; + } + softwire { + ipv4 193.5.1.101; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:dc; + } + softwire { + ipv4 193.5.1.102; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:112; + } + softwire { + ipv4 193.5.1.102; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:124; + } + softwire { + ipv4 193.5.1.100; + psid 19; + b4-ipv6 fc00:1:2:3:4:5:0:90; + } + softwire { + ipv4 193.5.1.101; + psid 37; + b4-ipv6 fc00:1:2:3:4:5:0:e1; + } + softwire { + ipv4 193.5.1.101; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:d5; + } + softwire { + ipv4 193.5.1.100; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:a3; + } + softwire { + ipv4 193.5.1.100; + psid 23; + b4-ipv6 fc00:1:2:3:4:5:0:94; + } + softwire { + ipv4 193.5.1.100; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:a5; + } + softwire { + ipv4 193.5.1.100; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:7e; + } + softwire { + ipv4 193.5.1.102; + psid 55; + b4-ipv6 fc00:1:2:3:4:5:0:132; + } + softwire { + ipv4 193.5.1.101; + psid 40; + b4-ipv6 fc00:1:2:3:4:5:0:e4; + } + softwire { + ipv4 193.5.1.102; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:10a; + } + softwire { + ipv4 193.5.1.101; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:c2; + } + softwire { + ipv4 193.5.1.100; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:99; + } + softwire { + ipv4 193.5.1.101; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:e5; + } + softwire { + ipv4 193.5.1.101; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:c3; + } + softwire { + ipv4 193.5.1.101; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:fa; + } + softwire { + ipv4 193.5.1.102; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:128; + } + softwire { + ipv4 193.5.1.102; + psid 31; + b4-ipv6 fc00:1:2:3:4:5:0:11a; + } + softwire { + ipv4 193.5.1.101; + psid 13; + b4-ipv6 fc00:1:2:3:4:5:0:c9; + } + softwire { + ipv4 193.5.1.101; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:f7; + } + softwire { + ipv4 193.5.1.101; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:f5; + } + softwire { + ipv4 193.5.1.102; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:106; + } + softwire { + ipv4 193.5.1.101; + psid 46; + b4-ipv6 fc00:1:2:3:4:5:0:ea; + } + softwire { + ipv4 193.5.1.102; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:11d; + } + softwire { + ipv4 193.5.1.100; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:7f; + } + softwire { + ipv4 193.5.1.101; + psid 30; + b4-ipv6 fc00:1:2:3:4:5:0:da; + } + softwire { + ipv4 193.5.1.101; + psid 45; + b4-ipv6 fc00:1:2:3:4:5:0:e9; + } + softwire { + ipv4 193.5.1.101; + psid 61; + b4-ipv6 fc00:1:2:3:4:5:0:f9; + } + softwire { + ipv4 193.5.1.102; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:10c; + } + softwire { + ipv4 193.5.1.101; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:d9; + } + softwire { + ipv4 193.5.1.102; + psid 33; + b4-ipv6 fc00:1:2:3:4:5:0:11c; + } + softwire { + ipv4 193.5.1.101; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:c8; + } + softwire { + ipv4 193.5.1.101; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:c0; + } + softwire { + ipv4 193.5.1.101; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:f0; + } + softwire { + ipv4 193.5.1.102; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:10d; + } + softwire { + ipv4 193.5.1.101; + psid 3; + b4-ipv6 fc00:1:2:3:4:5:0:bf; + } + softwire { + ipv4 193.5.1.102; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:105; + } + softwire { + ipv4 193.5.1.100; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:85; + } + softwire { + ipv4 193.5.1.101; + psid 2; + b4-ipv6 fc00:1:2:3:4:5:0:be; + } + softwire { + ipv4 193.5.1.100; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:a4; + } + softwire { + ipv4 193.5.1.101; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:e0; + } + softwire { + ipv4 193.5.1.102; + psid 32; + b4-ipv6 fc00:1:2:3:4:5:0:11b; + } + softwire { + ipv4 193.5.1.102; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:127; + } + softwire { + ipv4 193.5.1.102; + psid 48; + b4-ipv6 fc00:1:2:3:4:5:0:12b; + } + softwire { + ipv4 193.5.1.102; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:10f; + } + softwire { + ipv4 193.5.1.100; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:8b; + } + softwire { + ipv4 193.5.1.100; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:95; + } + softwire { + ipv4 193.5.1.102; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:12e; + } + softwire { + ipv4 193.5.1.102; + psid 25; + b4-ipv6 fc00:1:2:3:4:5:0:114; + } + softwire { + ipv4 193.5.1.100; + psid 10; + b4-ipv6 fc00:1:2:3:4:5:0:87; + } + softwire { + ipv4 193.5.1.100; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:93; + } + softwire { + ipv4 193.5.1.102; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:103; + } + softwire { + ipv4 193.5.1.102; + psid 22; + b4-ipv6 fc00:1:2:3:4:5:0:111; + } + softwire { + ipv4 193.5.1.101; + psid 56; + b4-ipv6 fc00:1:2:3:4:5:0:f4; + } + softwire { + ipv4 193.5.1.100; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:98; + } + softwire { + ipv4 193.5.1.100; + psid 63; + b4-ipv6 fc00:1:2:3:4:5:0:bc; + } + softwire { + ipv4 193.5.1.102; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:115; + } + softwire { + ipv4 193.5.1.100; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:b3; + } + softwire { + ipv4 193.5.1.100; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:0:a9; + } + softwire { + ipv4 193.5.1.102; + psid 50; + b4-ipv6 fc00:1:2:3:4:5:0:12d; + } + softwire { + ipv4 193.5.1.100; + psid 9; + b4-ipv6 fc00:1:2:3:4:5:0:86; + } + softwire { + ipv4 193.5.1.101; + psid 58; + b4-ipv6 fc00:1:2:3:4:5:0:f6; + } + softwire { + ipv4 193.5.1.100; + psid 18; + b4-ipv6 fc00:1:2:3:4:5:0:8f; + } + softwire { + ipv4 193.5.1.101; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:d8; + } + softwire { + ipv4 193.5.1.101; + psid 27; + b4-ipv6 fc00:1:2:3:4:5:0:d7; + } + softwire { + ipv4 193.5.1.102; + psid 54; + b4-ipv6 fc00:1:2:3:4:5:0:131; + } + softwire { + ipv4 193.5.1.102; + psid 59; + b4-ipv6 fc00:1:2:3:4:5:0:136; + } + softwire { + ipv4 193.5.1.100; + psid 41; + b4-ipv6 fc00:1:2:3:4:5:0:a6; + } + softwire { + ipv4 193.5.1.102; + psid 12; + b4-ipv6 fc00:1:2:3:4:5:0:107; + } + softwire { + ipv4 193.5.1.100; + psid 35; + b4-ipv6 fc00:1:2:3:4:5:0:a0; + } + softwire { + ipv4 193.5.1.101; + psid 11; + b4-ipv6 fc00:1:2:3:4:5:0:c7; + } + softwire { + ipv4 193.5.1.100; + psid 47; + b4-ipv6 fc00:1:2:3:4:5:0:ac; + } + softwire { + ipv4 193.5.1.101; + psid 24; + b4-ipv6 fc00:1:2:3:4:5:0:d4; + } + softwire { + ipv4 193.5.1.102; + psid 1; + b4-ipv6 fc00:1:2:3:4:5:0:fc; + } + softwire { + ipv4 193.5.1.101; + psid 8; + b4-ipv6 fc00:1:2:3:4:5:0:c4; + } + softwire { + ipv4 193.5.1.101; + psid 14; + b4-ipv6 fc00:1:2:3:4:5:0:ca; + } + softwire { + ipv4 193.5.1.101; + psid 17; + b4-ipv6 fc00:1:2:3:4:5:0:cd; + } + softwire { + ipv4 193.5.1.102; + psid 7; + b4-ipv6 fc00:1:2:3:4:5:0:102; + } + softwire { + ipv4 193.5.1.101; + psid 51; + b4-ipv6 fc00:1:2:3:4:5:0:ef; + } + softwire { + ipv4 193.5.1.102; + psid 6; + b4-ipv6 fc00:1:2:3:4:5:0:101; + } + softwire { + ipv4 193.5.1.102; + psid 62; + b4-ipv6 fc00:1:2:3:4:5:0:139; + } + softwire { + ipv4 193.5.1.102; + psid 28; + b4-ipv6 fc00:1:2:3:4:5:0:117; + } + softwire { + ipv4 193.5.1.100; + psid 16; + b4-ipv6 fc00:1:2:3:4:5:0:8d; + } + softwire { + ipv4 193.5.1.100; + psid 29; + b4-ipv6 fc00:1:2:3:4:5:0:9a; + } + softwire { + ipv4 193.5.1.102; + psid 42; + b4-ipv6 fc00:1:2:3:4:5:0:125; + } + softwire { + ipv4 193.5.1.100; + psid 21; + b4-ipv6 fc00:1:2:3:4:5:0:92; + } + softwire { + ipv4 193.5.1.102; + psid 49; + b4-ipv6 fc00:1:2:3:4:5:0:12c; + } + softwire { + ipv4 193.5.1.100; + psid 57; + b4-ipv6 fc00:1:2:3:4:5:0:b6; + } + softwire { + ipv4 193.5.1.101; + psid 20; + b4-ipv6 fc00:1:2:3:4:5:0:d0; + } + softwire { + ipv4 193.5.1.102; + psid 52; + b4-ipv6 fc00:1:2:3:4:5:0:12f; + } + softwire { + ipv4 193.5.1.100; + psid 26; + b4-ipv6 fc00:1:2:3:4:5:0:97; + } + softwire { + ipv4 193.5.1.102; + psid 36; + b4-ipv6 fc00:1:2:3:4:5:0:11f; + } + softwire { + ipv4 193.5.1.100; + psid 4; + b4-ipv6 fc00:1:2:3:4:5:0:81; + } + softwire { + ipv4 193.5.1.101; + psid 34; + b4-ipv6 fc00:1:2:3:4:5:0:de; + } + softwire { + ipv4 193.5.1.100; + psid 5; + b4-ipv6 fc00:1:2:3:4:5:0:82; + } + softwire { + ipv4 193.5.1.101; + psid 38; + b4-ipv6 fc00:1:2:3:4:5:0:e2; + } + softwire { + ipv4 193.5.1.101; + psid 15; + b4-ipv6 fc00:1:2:3:4:5:0:cb; + } + softwire { + ipv4 193.5.1.101; + psid 39; + b4-ipv6 fc00:1:2:3:4:5:0:e3; + } + softwire { + ipv4 193.5.1.102; + psid 53; + b4-ipv6 fc00:1:2:3:4:5:0:130; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.0.1.1; + mac 02:aa:aa:aa:aa:aa; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip fc00::100; + mac 02:aa:aa:aa:aa:aa; + mtu 9500; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf index 5382cf1772..2b9602cd94 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf @@ -1,63 +1,65 @@ -binding-table { - br-address 2a02:587:f700::100; - psid-map { - addr 10.10.0.10; - psid-length 6; - shift 10; +softwire-config { + binding-table { + br-address 2a02:587:f700::100; + psid-map { + addr 10.10.0.10; + psid-length 6; + shift 10; + } + psid-map { + addr 10.10.0.0; + end-addr 10.10.0.1; + psid-length 6; + shift 10; + } + softwire { + ipv4 10.10.0.0; + psid 1; + b4-ipv6 2a02:587:f710::400; + } + softwire { + ipv4 10.10.0.0; + psid 4; + b4-ipv6 2a02:587:f710::430; + } + softwire { + ipv4 10.10.0.0; + psid 2; + b4-ipv6 2a02:587:f710::410; + } + softwire { + ipv4 10.10.0.0; + psid 3; + b4-ipv6 2a02:587:f710::420; + } } - psid-map { - addr 10.10.0.0; - end-addr 10.10.0.1; - psid-length 6; - shift 10; + external-interface { + error-rate-limiting { + packets 600000; + } + ip 192.168.10.2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + } } - softwire { - ipv4 10.10.0.0; - psid 1; - b4-ipv6 2a02:587:f710::400; - } - softwire { - ipv4 10.10.0.0; - psid 4; - b4-ipv6 2a02:587:f710::430; - } - softwire { - ipv4 10.10.0.0; - psid 2; - b4-ipv6 2a02:587:f710::410; - } - softwire { - ipv4 10.10.0.0; - psid 3; - b4-ipv6 2a02:587:f710::420; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 192.168.10.2; - mac 02:cf:69:15:81:01; - mtu 9000; - next-hop { - mac 90:e2:ba:94:2a:bc; - } - reassembly { - max-fragments-per-packet 40; - } -} -internal-interface { - error-rate-limiting { - packets 600000; - } - hairpinning false; - ip fc00:168:10::2; - mac 02:cf:69:15:81:01; - mtu 9000; - next-hop { - mac 90:e2:ba:94:2a:bc; - } - reassembly { - max-fragments-per-packet 40; + internal-interface { + error-rate-limiting { + packets 600000; + } + hairpinning false; + ip fc00:168:10::2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + } } } diff --git a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf index dd22d885b9..2e083887ca 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf @@ -1,65 +1,67 @@ -binding-table { - br-address 2a02:587:f700::100; - psid-map { - addr 10.10.0.10; - psid-length 6; - shift 10; +softwire-config { + binding-table { + br-address 2a02:587:f700::100; + psid-map { + addr 10.10.0.10; + psid-length 6; + shift 10; + } + psid-map { + addr 10.10.0.0; + end-addr 10.10.0.1; + psid-length 6; + shift 10; + } + softwire { + ipv4 10.10.0.0; + psid 1; + b4-ipv6 2a02:587:f710::400; + } + softwire { + ipv4 10.10.0.0; + psid 4; + b4-ipv6 2a02:587:f710::430; + } + softwire { + ipv4 10.10.0.0; + psid 2; + b4-ipv6 2a02:587:f710::410; + } + softwire { + ipv4 10.10.0.0; + psid 3; + b4-ipv6 2a02:587:f710::420; + } } - psid-map { - addr 10.10.0.0; - end-addr 10.10.0.1; - psid-length 6; - shift 10; + external-interface { + error-rate-limiting { + packets 600000; + } + ip 192.168.10.2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; } - softwire { - ipv4 10.10.0.0; - psid 1; - b4-ipv6 2a02:587:f710::400; + internal-interface { + error-rate-limiting { + packets 600000; + } + hairpinning false; + ip fc00:168:10::2; + mac 02:cf:69:15:81:01; + mtu 9000; + next-hop { + mac 90:e2:ba:94:2a:bc; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; } - softwire { - ipv4 10.10.0.0; - psid 4; - b4-ipv6 2a02:587:f710::430; - } - softwire { - ipv4 10.10.0.0; - psid 2; - b4-ipv6 2a02:587:f710::410; - } - softwire { - ipv4 10.10.0.0; - psid 3; - b4-ipv6 2a02:587:f710::420; - } -} -external-interface { - error-rate-limiting { - packets 600000; - } - ip 192.168.10.2; - mac 02:cf:69:15:81:01; - mtu 9000; - next-hop { - mac 90:e2:ba:94:2a:bc; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1092; -} -internal-interface { - error-rate-limiting { - packets 600000; - } - hairpinning false; - ip fc00:168:10::2; - mac 02:cf:69:15:81:01; - mtu 9000; - next-hop { - mac 90:e2:ba:94:2a:bc; - } - reassembly { - max-fragments-per-packet 40; - } - vlan-tag 1638; } From 2beb5ee6674e08effb3174009883635fcb9b7435 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 12:51:13 +0100 Subject: [PATCH 348/631] Update binding-table update fast-path for schema change --- src/apps/config/support/snabb-softwire-v1.lua | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index d88be3ca8b..71f225412c 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -25,7 +25,8 @@ local function get_softwire_grammar() local schema = yang.load_schema_by_name('snabb-softwire-v1') local grammar = data.data_grammar_from_schema(schema) softwire_grammar = - assert(grammar.members['binding-table'].members['softwire']) + assert(grammar.members['softwire-config']. + members['binding-table'].members['softwire']) end return softwire_grammar end @@ -42,9 +43,10 @@ end local function compute_config_actions(old_graph, new_graph, to_restart, verb, path, arg) - if verb == 'add' and path == '/binding-table/softwire' then + if verb == 'add' and path == '/softwire-config/binding-table/softwire' then return add_softwire_entry_actions(new_graph, arg) - elseif verb == 'remove' and path:match('^/binding%-table/softwire') then + elseif (verb == 'remove' and + path:match('^/softwire%-config/binding%-table/softwire')) then return remove_softwire_entry_actions(new_graph, path) else return generic.compute_config_actions( @@ -54,9 +56,10 @@ end local function update_mutable_objects_embedded_in_app_initargs( in_place_dependencies, app_graph, schema_name, verb, path, arg) - if verb == 'add' and path == '/binding-table/softwire' then + if verb == 'add' and path == '/softwire-config/binding-table/softwire' then return in_place_dependencies - elseif verb == 'remove' and path:match('^/binding%-table/softwire') then + elseif (verb == 'remove' and + path:match('^/softwire%-config/binding%-table/softwire')) then return in_place_dependencies else return generic.update_mutable_objects_embedded_in_app_initargs( @@ -66,9 +69,10 @@ end local function compute_apps_to_restart_after_configuration_update( schema_name, configuration, verb, path, in_place_dependencies) - if verb == 'add' and path == '/binding-table/softwire' then + if verb == 'add' and path == '/softwire-config/binding-table/softwire' then return {} - elseif verb == 'remove' and path:match('^/binding%-table/softwire') then + elseif (verb == 'remove' and + path:match('^/softwire%-config/binding%-table/softwire')) then return {} else return generic.compute_apps_to_restart_after_configuration_update( From e5a69159bb88e0275d52e3a37054cd14ed09e078 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 13:13:07 +0100 Subject: [PATCH 349/631] Fix up data-plane fallout from lwaftr schema change Since the lwaftr just uses the configuration as given to it by the YANG configuration parser, a change in the schema or a change in the way that schema maps to data requires corresponding data-plane changes. This patch fixes up the data plane for the config changes. --- src/apps/lwaftr/lwaftr.lua | 1 + src/program/lwaftr/check/check.lua | 2 - src/program/lwaftr/setup.lua | 170 ++++++++++++++----------- src/program/snabbvmx/lwaftr/lwaftr.lua | 30 ++--- src/program/snabbvmx/lwaftr/setup.lua | 52 ++++---- 5 files changed, 138 insertions(+), 117 deletions(-) diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index 7a26c96e9d..d179c19a3a 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -242,6 +242,7 @@ LwAftr = { yang_schema = 'snabb-softwire-v1' } function LwAftr:new(conf) if conf.debug then debug = true end local o = setmetatable({}, {__index=LwAftr}) + conf = conf.softwire_config o.conf = conf o.binding_table = bt.load(conf.binding_table) diff --git a/src/program/lwaftr/check/check.lua b/src/program/lwaftr/check/check.lua index 0014ef2a9e..f936604daf 100644 --- a/src/program/lwaftr/check/check.lua +++ b/src/program/lwaftr/check/check.lua @@ -41,8 +41,6 @@ function run(args) local conf_file, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap, counters_path = unpack(args) local conf = lwconf.load_lwaftr_config(conf_file) - for k,v in pairs(conf) do print(k, v) end - local c = config.new() load_check(c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap) engine.configure(c) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index af072850a7..a59ddc0590 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -24,62 +24,64 @@ end function lwaftr_app(c, conf) assert(type(conf) == 'table') + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface local function append(t, elem) table.insert(t, elem) end local function prepend(t, elem) table.insert(t, 1, elem) end config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = - conf.external_interface.reassembly.max_packets, + external_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - conf.external_interface.reassembly.max_fragments_per_packet }) + external_interface.reassembly.max_fragments_per_packet }) config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { max_ipv6_reassembly_packets = - conf.internal_interface.reassembly.max_packets, + internal_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - conf.internal_interface.reassembly.max_fragments_per_packet }) + internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "icmpechov4", ipv4_apps.ICMPEcho, - { address = convert_ipv4(conf.external_interface.ip) }) + { address = convert_ipv4(external_interface.ip) }) config.app(c, "icmpechov6", ipv6_apps.ICMPEcho, - { address = conf.internal_interface.ip }) + { address = internal_interface.ip }) config.app(c, 'lwaftr', lwaftr.LwAftr, conf) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, - { mtu=conf.external_interface.mtu }) + { mtu=external_interface.mtu }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, - { mtu=conf.internal_interface.mtu }) + { mtu=internal_interface.mtu }) config.app(c, "ndp", ipv6_apps.NDP, - { src_ipv6 = conf.internal_interface.ip, - src_eth = conf.internal_interface.mac, - dst_eth = conf.internal_interface.next_hop.mac, - dst_ipv6 = conf.internal_interface.next_hop.ip }) + { src_ipv6 = internal_interface.ip, + src_eth = internal_interface.mac, + dst_eth = internal_interface.next_hop.mac, + dst_ipv6 = internal_interface.next_hop.ip }) config.app(c, "arp", ipv4_apps.ARP, - { src_ipv4 = convert_ipv4(conf.external_interface.ip), - src_eth = conf.external_interface.mac, - dst_eth = conf.external_interface.next_hop.mac, - dst_ipv4 = convert_ipv4(conf.external_interface.next_hop.ip) }) + { src_ipv4 = convert_ipv4(external_interface.ip), + src_eth = external_interface.mac, + dst_eth = external_interface.next_hop.mac, + dst_ipv4 = convert_ipv4(external_interface.next_hop.ip) }) local preprocessing_apps_v4 = { "reassemblerv4" } local preprocessing_apps_v6 = { "reassemblerv6" } local postprocessing_apps_v4 = { "fragmenterv4" } local postprocessing_apps_v6 = { "fragmenterv6" } - if conf.external_interface.ingress_filter then + if external_interface.ingress_filter then config.app(c, "ingress_filterv4", PcapFilter, - { filter = conf.external_interface.ingress_filter }) + { filter = external_interface.ingress_filter }) append(preprocessing_apps_v4, "ingress_filterv4") end - if conf.internal_interface.ingress_filter then + if internal_interface.ingress_filter then config.app(c, "ingress_filterv6", PcapFilter, - { filter = conf.internal_interface.ingress_filter }) + { filter = internal_interface.ingress_filter }) append(preprocessing_apps_v6, "ingress_filterv6") end - if conf.external_interface.egress_filter then + if external_interface.egress_filter then config.app(c, "egress_filterv4", PcapFilter, - { filter = conf.external_interface.egress_filter }) + { filter = external_interface.egress_filter }) prepend(postprocessing_apps_v4, "egress_filterv4") end - if conf.internal_interface.egress_filter then + if internal_interface.egress_filter then config.app(c, "egress_filterv6", PcapFilter, - { filter = conf.internal_interface.egress_filter }) + { filter = internal_interface.egress_filter }) prepend(postprocessing_apps_v6, "egress_filterv6") end @@ -148,19 +150,21 @@ end function load_phy(c, conf, v4_nic_name, v4_nic_pci, v6_nic_name, v6_nic_pci) lwaftr_app(c, conf) + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface config.app(c, v4_nic_name, Intel82599, { pciaddr=v4_nic_pci, - vmdq=conf.external_interface.vlan_tag, - vlan=conf.external_interface.vlan_tag, + vmdq=external_interface.vlan_tag, + vlan=external_interface.vlan_tag, rxcounter=1, - macaddr=ethernet:ntop(conf.external_interface.mac)}) + macaddr=ethernet:ntop(external_interface.mac)}) config.app(c, v6_nic_name, Intel82599, { pciaddr=v6_nic_pci, - vmdq=conf.internal_interface.vlan_tag, - vlan=conf.internal_interface.vlan_tag, + vmdq=internal_interface.vlan_tag, + vlan=internal_interface.vlan_tag, rxcounter=1, - macaddr = ethernet:ntop(conf.internal_interface.mac)}) + macaddr = ethernet:ntop(internal_interface.mac)}) link_source(c, v4_nic_name..'.tx', v6_nic_name..'.tx') link_sink(c, v4_nic_name..'.rx', v6_nic_name..'.rx') @@ -170,13 +174,15 @@ function load_on_a_stick(c, conf, args) lwaftr_app(c, conf) local v4_nic_name, v6_nic_name, v4v6, pciaddr, mirror = args.v4_nic_name, args.v6_nic_name, args.v4v6, args.pciaddr, args.mirror + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface if v4v6 then config.app(c, 'nic', Intel82599, { pciaddr = pciaddr, - vmdq=conf.external_interface.vlan_tag, - vlan=conf.external_interface.vlan_tag, - macaddr = ethernet:ntop(conf.external_interface.mac)}) + vmdq=external_interface.vlan_tag, + vlan=external_interface.vlan_tag, + macaddr = ethernet:ntop(external_interface.mac)}) if mirror then local Tap = require("apps.tap.tap").Tap local ifname = mirror @@ -196,13 +202,13 @@ function load_on_a_stick(c, conf, args) else config.app(c, v4_nic_name, Intel82599, { pciaddr = pciaddr, - vmdq=conf.external_interface.vlan_tag, - vlan=conf.external_interface.vlan_tag, - macaddr = ethernet:ntop(conf.external_interface.mac)}) + vmdq=external_interface.vlan_tag, + vlan=external_interface.vlan_tag, + macaddr = ethernet:ntop(external_interface.mac)}) config.app(c, v6_nic_name, Intel82599, { pciaddr = pciaddr, - vmdq=conf.internal_interface.vlan_tag, - vlan=conf.internal_interface.vlan_tag, + vmdq=internal_interface.vlan_tag, + vlan=internal_interface.vlan_tag, macaddr = ethernet:ntop(conf.internal_interface.mac)}) link_source(c, v4_nic_name..'.tx', v6_nic_name..'.tx') @@ -212,15 +218,17 @@ end function load_virt(c, conf, v4_nic_name, v4_nic_pci, v6_nic_name, v6_nic_pci) lwaftr_app(c, conf) + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface config.app(c, v4_nic_name, VirtioNet, { pciaddr=v4_nic_pci, - vlan=conf.external_interface.vlan_tag, - macaddr=ethernet:ntop(conf.external_interface.mac)}) + vlan=external_interface.vlan_tag, + macaddr=ethernet:ntop(external_interface.mac)}) config.app(c, v6_nic_name, VirtioNet, { pciaddr=v6_nic_pci, - vlan=conf.internal_interface.vlan_tag, - macaddr = ethernet:ntop(conf.internal_interface.mac)}) + vlan=internal_interface.vlan_tag, + macaddr = ethernet:ntop(internal_interface.mac)}) link_source(c, v4_nic_name..'.tx', v6_nic_name..'.tx') link_sink(c, v4_nic_name..'.rx', v6_nic_name..'.rx') @@ -228,18 +236,20 @@ end function load_bench(c, conf, v4_pcap, v6_pcap, v4_sink, v6_sink) lwaftr_app(c, conf) + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface config.app(c, "capturev4", pcap.PcapReader, v4_pcap) config.app(c, "capturev6", pcap.PcapReader, v6_pcap) config.app(c, "repeaterv4", basic_apps.Repeater) config.app(c, "repeaterv6", basic_apps.Repeater) - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.app(c, "untagv4", vlan.Untagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) end - if conf.internal_interface.vlan_tag then + if internal_interface.vlan_tag then config.app(c, "untagv6", vlan.Untagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) end config.app(c, v4_sink, basic_apps.Sink) config.app(c, v6_sink, basic_apps.Sink) @@ -248,11 +258,11 @@ function load_bench(c, conf, v4_pcap, v6_pcap, v4_sink, v6_sink) config.link(c, "capturev6.output -> repeaterv6.input") local v4_src, v6_src = 'repeaterv4.output', 'repeaterv6.output' - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.link(c, v4_src.." -> untagv4.input") v4_src = "untagv4.output" end - if conf.internal_interface.vlan_tag then + if internal_interface.vlan_tag then config.link(c, v6_src.." -> untagv6.input") v6_src = "untagv6.output" end @@ -262,22 +272,24 @@ end function load_check_on_a_stick (c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap) lwaftr_app(c, conf) + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface config.app(c, "capturev4", pcap.PcapReader, inv4_pcap) config.app(c, "capturev6", pcap.PcapReader, inv6_pcap) config.app(c, "output_filev4", pcap.PcapWriter, outv4_pcap) config.app(c, "output_filev6", pcap.PcapWriter, outv6_pcap) - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.app(c, "untagv4", vlan.Untagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) config.app(c, "tagv4", vlan.Tagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) end - if conf.internal_interface.vlan_tag then + if internal_interface.vlan_tag then config.app(c, "untagv6", vlan.Untagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) config.app(c, "tagv6", vlan.Tagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) end config.app(c, 'v4v6', V4V6) @@ -287,7 +299,7 @@ function load_check_on_a_stick (c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6 local sources = { "v4v6.v4", "v4v6.v6" } local sinks = { "v4v6.v4", "v4v6.v6" } - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.link(c, "capturev4.output -> untagv4.input") config.link(c, "capturev6.output -> untagv6.input") config.link(c, "untagv4.output -> join.in1") @@ -313,28 +325,30 @@ end function load_check(c, conf, inv4_pcap, inv6_pcap, outv4_pcap, outv6_pcap) lwaftr_app(c, conf) + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface config.app(c, "capturev4", pcap.PcapReader, inv4_pcap) config.app(c, "capturev6", pcap.PcapReader, inv6_pcap) config.app(c, "output_filev4", pcap.PcapWriter, outv4_pcap) config.app(c, "output_filev6", pcap.PcapWriter, outv6_pcap) - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.app(c, "untagv4", vlan.Untagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) config.app(c, "tagv4", vlan.Tagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) end - if conf.internal_interface.vlan_tag then + if internal_interface.vlan_tag then config.app(c, "untagv6", vlan.Untagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) config.app(c, "tagv6", vlan.Tagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) end local sources = { "capturev4.output", "capturev6.output" } local sinks = { "output_filev4.input", "output_filev6.input" } - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then sources = { "untagv4.output", "untagv6.output" } sinks = { "tagv4.input", "tagv6.input" } @@ -350,23 +364,25 @@ end function load_soak_test(c, conf, inv4_pcap, inv6_pcap) lwaftr_app(c, conf) + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface config.app(c, "capturev4", pcap.PcapReader, inv4_pcap) config.app(c, "capturev6", pcap.PcapReader, inv6_pcap) config.app(c, "loop_v4", basic_apps.Repeater) config.app(c, "loop_v6", basic_apps.Repeater) config.app(c, "sink", basic_apps.Sink) - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.app(c, "untagv4", vlan.Untagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) config.app(c, "tagv4", vlan.Tagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) end - if conf.internal_interface.vlan_tag then + if internal_interface.vlan_tag then config.app(c, "untagv6", vlan.Untagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) config.app(c, "tagv6", vlan.Tagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) end local sources = { "loop_v4.output", "loop_v6.output" } @@ -375,7 +391,7 @@ function load_soak_test(c, conf, inv4_pcap, inv6_pcap) config.link(c, "capturev4.output -> loop_v4.input") config.link(c, "capturev6.output -> loop_v6.input") - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then sources = { "untagv4.output", "untagv6.output" } sinks = { "tagv4.input", "tagv6.input" } @@ -391,23 +407,25 @@ end function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) lwaftr_app(c, conf) + local external_interface = conf.softwire_config.external_interface + local internal_interface = conf.softwire_config.internal_interface config.app(c, "capturev4", pcap.PcapReader, inv4_pcap) config.app(c, "capturev6", pcap.PcapReader, inv6_pcap) config.app(c, "loop_v4", basic_apps.Repeater) config.app(c, "loop_v6", basic_apps.Repeater) config.app(c, "sink", basic_apps.Sink) - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.app(c, "untagv4", vlan.Untagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) config.app(c, "tagv4", vlan.Tagger, - { tag=conf.external_interface.vlan_tag }) + { tag=external_interface.vlan_tag }) end - if conf.internal_interface.vlan_tag then + if internal_interface.vlan_tag then config.app(c, "untagv6", vlan.Untagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) config.app(c, "tagv6", vlan.Tagger, - { tag=conf.internal_interface.vlan_tag }) + { tag=internal_interface.vlan_tag }) end config.app(c, 'v4v6', V4V6) @@ -420,7 +438,7 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) config.link(c, "capturev4.output -> loop_v4.input") config.link(c, "capturev6.output -> loop_v6.input") - if conf.external_interface.vlan_tag then + if external_interface.vlan_tag then config.link(c, "loop_v4.output -> untagv4.input") config.link(c, "loop_v6.output -> untagv6.input") config.link(c, "untagv4.output -> join.in1") diff --git a/src/program/snabbvmx/lwaftr/lwaftr.lua b/src/program/snabbvmx/lwaftr/lwaftr.lua index d7c9e9559e..61ed8558aa 100644 --- a/src/program/snabbvmx/lwaftr/lwaftr.lua +++ b/src/program/snabbvmx/lwaftr/lwaftr.lua @@ -72,16 +72,16 @@ function parse_args (args) return opts, conf_file, id, pci, mac, sock_path, mirror_id end -local function effective_vlan (conf, lwconf) +local function effective_vlan (conf, external_interface, internal_interface) if conf.settings and conf.settings.vlan then return conf.settings.vlan end - if lwconf.external_interface.vlan_tag then - if lwconf.external_interface.vlan_tag == lwconf.internal_interface.vlan_tag then - return lwconf.external_interface.vlan_tag + if external_interface.vlan_tag then + if external_interface.vlan_tag == internal_interface.vlan_tag then + return external_interface.vlan_tag end - return {v4_vlan_tag = lwconf.external_interface.vlan_tag, - v6_vlan_tag = lwconf.internal_interface.vlan_tag} + return {v4_vlan_tag = external_interface.vlan_tag, + v6_vlan_tag = internal_interface.vlan_tag} end return false end @@ -90,6 +90,7 @@ function run(args) local opts, conf_file, id, pci, mac, sock_path, mirror_id = parse_args(args) local conf, lwconf + local external_interface, internal_interface local ring_buffer_size = 2048 local ingress_drop_action = "flush" @@ -107,9 +108,10 @@ function run(args) fatal(("lwAFTR conf file '%s' not found"):format(conf.lwaftr)) end lwconf = require('apps.lwaftr.conf').load_lwaftr_config(conf.lwaftr) + external_interface = lwconf.softwire_config.external_interface + internal_interface = lwconf.softwire_config.internal_interface -- If one interface has vlan tags, the other one should as well. - assert((not lwconf.external_interface.vlan_tag) == - (not lwconf.internal_interface.vlan_tag)) + assert((not external_interface.vlan_tag) == (not internal_interface.vlan_tag)) else print(("Interface '%s' set to passthrough mode."):format(id)) ring_buffer_size = 1024 @@ -144,15 +146,11 @@ function run(args) local vlan = false local mtu = DEFAULT_MTU if lwconf then - vlan = effective_vlan(conf, lwconf) - mtu = lwconf.internal_interface.mtu - if lwconf.external_interface.mtu > mtu then - mtu = lwconf.external_interface.mtu - end + vlan = effective_vlan(conf, external_interface, internal_interface) + mtu = internal_interface.mtu + if external_interface.mtu > mtu then mtu = external_interface.mtu end mtu = mtu + constants.ethernet_header_size - if lwconf.external_interface.vlan_tag then - mtu = mtu + 4 - end + if external_interface.vlan_tag then mtu = mtu + 4 end end conf.interface = { diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index 18aa48041f..87331b59c8 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -41,6 +41,8 @@ local function load_driver (pciaddr) end local function load_virt (c, nic_id, lwconf, interface) + local external_interface = lwconf.softwire_config.external_interface + local internal_interface = lwconf.softwire_config.internal_interface assert(type(interface) == 'table') assert(nic_exists(interface.pci), "Couldn't find NIC: "..interface.pci) local driver = assert(load_driver(interface.pci)) @@ -49,8 +51,8 @@ local function load_virt (c, nic_id, lwconf, interface) print(("%s ether %s"):format(nic_id, interface.mac_address)) local v4_nic_name, v6_nic_name = nic_id..'_v4', nic_id..'v6' - local v4_mtu = lwconf.external_interface.mtu + constants.ethernet_header_size - if lwconf.external_interface.vlan_tag then + local v4_mtu = external_interface.mtu + constants.ethernet_header_size + if external_interface.vlan_tag then v4_mtu = v4_mtu + 4 end print(("Setting %s interface MTU to %d"):format(v4_nic_name, v4_mtu)) @@ -58,10 +60,10 @@ local function load_virt (c, nic_id, lwconf, interface) pciaddr = interface.pci, vmdq = interface.vlan and true, vlan = interface.vlan and interface.vlan.v4_vlan_tag, - macaddr = ethernet:ntop(lwconf.external_interface.mac), + macaddr = ethernet:ntop(external_interface.mac), mtu = v4_mtu }) - local v6_mtu = lwconf.internal_interface.mtu + constants.ethernet_header_size - if lwconf.internal_interface.vlan_tag then + local v6_mtu = internal_interface.mtu + constants.ethernet_header_size + if internal_interface.vlan_tag then v6_mtu = v6_mtu + 4 end print(("Setting %s interface MTU to %d"):format(v6_nic_name, v6_mtu)) @@ -69,7 +71,7 @@ local function load_virt (c, nic_id, lwconf, interface) pciaddr = interface.pci, vmdq = interface.vlan and true, vlan = interface.vlan and interface.vlan.v6_vlan_tag, - macaddr = ethernet:ntop(lwconf.internal_interface.mac), + macaddr = ethernet:ntop(internal_interface.mac), mtu = v6_mtu}) return v4_nic_name, v6_nic_name @@ -116,23 +118,25 @@ local function load_phy (c, nic_id, interface) return chain_input, chain_output end -local function requires_splitter (lwconf) - if not lwconf.internal_interface.vlan_tag then return true end - return lwconf.internal_interface.vlan_tag == lwconf.external_interface.vlan_tag +local function requires_splitter (internal_interface, external_interface) + if not internal_interface.vlan_tag then return true end + return internal_interface.vlan_tag == external_interface.vlan_tag end function lwaftr_app(c, conf, lwconf, sock_path) assert(type(conf) == 'table') assert(type(lwconf) == 'table') + local external_interface = lwconf.softwire_config.external_interface + local internal_interface = lwconf.softwire_config.internal_interface - print(("Hairpinning: %s"):format(yesno(lwconf.internal_interface.hairpinning))) + print(("Hairpinning: %s"):format(yesno(internal_interface.hairpinning))) local virt_id = "vm_" .. conf.interface.id local phy_id = "nic_" .. conf.interface.id local chain_input, chain_output local v4_input, v4_output, v6_input, v6_output - local use_splitter = requires_splitter(lwconf) + local use_splitter = requires_splitter(internal_interface, external_interface) if not use_splitter then local v4, v6 = load_virt(c, phy_id, lwconf, conf.interface) v4_output, v6_output = v4..".tx", v6..".tx" @@ -166,12 +170,12 @@ function lwaftr_app(c, conf, lwconf, sock_path) print(("IPv6 fragmentation and reassembly: %s"):format(yesno( conf.ipv6_interface.fragmentation))) if conf.ipv6_interface.fragmentation then - local mtu = conf.ipv6_interface.mtu or lwconf.internal_interface.mtu + local mtu = conf.ipv6_interface.mtu or internal_interface.mtu config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { max_ipv6_reassembly_packets = - lwconf.internal_interface.reassembly.max_packets, + internal_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - lwconf.internal_interface.reassembly.max_fragments_per_packet + internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, { mtu = mtu, @@ -201,12 +205,12 @@ function lwaftr_app(c, conf, lwconf, sock_path) print(("IPv4 fragmentation and reassembly: %s"):format(yesno( conf.ipv4_interface.fragmentation))) if conf.ipv4_interface.fragmentation then - local mtu = conf.ipv4_interface.mtu or lwconf.external_interface.mtu + local mtu = conf.ipv4_interface.mtu or external_interface.mtu config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = - lwconf.external_interface.reassembly.max_packets, + external_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - lwconf.external_interface.reassembly.max_fragments_per_packet + external_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, { mtu = mtu @@ -319,18 +323,20 @@ end local function lwaftr_app_check (c, conf, lwconf, sources, sinks) assert(type(conf) == "table") assert(type(lwconf) == "table") + local external_interface = lwconf.softwire_config.external_interface + local internal_interface = lwconf.softwire_config.internal_interface local v4_src, v6_src = unpack(sources) local v4_sink, v6_sink = unpack(sinks) if conf.ipv6_interface then if conf.ipv6_interface.fragmentation then - local mtu = conf.ipv6_interface.mtu or lwconf.internal_interface.mtu + local mtu = conf.ipv6_interface.mtu or internal_interface.mtu config.app(c, "reassemblerv6", ipv6_apps.ReassembleV6, { max_ipv6_reassembly_packets = - lwconf.internal_interface.reassembly.max_packets, + internal_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - lwconf.internal_interface.reassembly.max_fragments_per_packet + internal_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv6", ipv6_apps.Fragmenter, { mtu = mtu, @@ -355,12 +361,12 @@ local function lwaftr_app_check (c, conf, lwconf, sources, sinks) if conf.ipv4_interface then if conf.ipv4_interface.fragmentation then - local mtu = conf.ipv4_interface.mtu or lwconf.external_interface.mtu + local mtu = conf.ipv4_interface.mtu or external_interface.mtu config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = - lwconf.external_interface.reassembly.max_packets, + external_interface.reassembly.max_packets, max_fragments_per_reassembly_packet = - lwconf.external_interface.reassembly.max_fragments_per_packet + external_interface.reassembly.max_fragments_per_packet }) config.app(c, "fragmenterv4", ipv4_apps.Fragmenter, { mtu = mtu From 84b8f6d425918e14fb38173c6de3f9d91c0a47d2 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 13:15:26 +0100 Subject: [PATCH 350/631] Fix up legacy lwaftr migrate-configuration --- .../migrate_configuration.lua | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index ad1fc0a576..0edd525a57 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -332,12 +332,14 @@ local function migrate_conf(old) end return { - external_interface = external, - internal_interface = internal, - binding_table = { - psid_map = psid_map, - br_address = old_bt.br_addresses, - softwire = old_bt.softwires + softwire_config = { + external_interface = external, + internal_interface = internal, + binding_table = { + psid_map = psid_map, + br_address = old_bt.br_addresses, + softwire = old_bt.softwires + } } } end From fa65d1ee3b80d6269c992c7ef275d5b20ac0608f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 13:24:33 +0100 Subject: [PATCH 351/631] Fix binding-table self test --- src/apps/lwaftr/binding_table.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index f388af1346..a6958c06d2 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -273,7 +273,8 @@ function selftest() local data = require('lib.yang.data') local schema = yang.load_schema_by_name('snabb-softwire-v1') local grammar = data.data_grammar_from_schema(schema) - local subgrammar = assert(grammar.members['binding-table']) + local subgrammar = assert(grammar.members['softwire-config']) + local subgrammar = assert(subgrammar.members['binding-table']) local parse = data.data_parser_from_grammar(subgrammar) return load(parse(str, '[test suite]')) end From 3733183ad2c8bf72f67d457780ff2ba6459a776b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 15:15:47 +0100 Subject: [PATCH 352/631] Attempt to fix snabbvmx selftest --- src/program/snabbvmx/lwaftr/setup.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index 87331b59c8..0aa3b4be1e 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -235,7 +235,7 @@ function lwaftr_app(c, conf, lwconf, sock_path) end end - if conf.ipv4_interface and conf.ipv6_interface and conf.preloaded_binding_table then + if conf.ipv4_interface and conf.ipv6_interface and lwconf then print("lwAFTR service: enabled") config.app(c, "nh_fwd6", nh_fwd.nh_fwd6, subset(nh_fwd.nh_fwd6.config, conf.ipv6_interface)) From 0cf1564feab8e4a85009ebc955a5f7de8d21bff2 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 15:33:36 +0100 Subject: [PATCH 353/631] Clean up snabbvmx setup fix --- src/program/snabbvmx/lwaftr/setup.lua | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/program/snabbvmx/lwaftr/setup.lua b/src/program/snabbvmx/lwaftr/setup.lua index 0aa3b4be1e..e1a105dd0f 100644 --- a/src/program/snabbvmx/lwaftr/setup.lua +++ b/src/program/snabbvmx/lwaftr/setup.lua @@ -235,7 +235,7 @@ function lwaftr_app(c, conf, lwconf, sock_path) end end - if conf.ipv4_interface and conf.ipv6_interface and lwconf then + if conf.ipv4_interface and conf.ipv6_interface then print("lwAFTR service: enabled") config.app(c, "nh_fwd6", nh_fwd.nh_fwd6, subset(nh_fwd.nh_fwd6.config, conf.ipv6_interface)) @@ -258,8 +258,7 @@ function lwaftr_app(c, conf, lwconf, sock_path) -- Add a special hairpinning queue to the lwaftr app. config.link(c, "lwaftr.hairpin_out -> lwaftr.hairpin_in") else - io.write("lwAFTR service: disabled ") - print("(either empty binding_table or v6 or v4 interface config missing)") + print("lwAFTR service: disabled (v6 or v4 interface config missing)") end if conf.ipv4_interface or conf.ipv6_interface then From 56bd3047a57801251f452654fc9f75bb4d401ea4 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 2 Dec 2016 17:16:27 +0100 Subject: [PATCH 354/631] Added bench support for --reconfigurable --- src/program/lwaftr/bench/bench.lua | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index a5cfcb2e96..49df4f048a 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -25,17 +25,25 @@ function parse_args(args) if not cpu or cpu ~= math.floor(cpu) or cpu < 0 then fatal("Invalid cpu number: "..arg) end - S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) - local wanted_node = numa.cpu_get_numa_node(cpu) - numa.bind_to_numa_node(wanted_node) - print("Bound to numa node: ", wanted_node) + + if opts.reconfigurable then + S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) + local wanted_node = numa.cpu_get_numa_node(cpu) + numa.bind_to_numa_node(wanted_node) + print("Bound to numa node:", wanted_node) + else + print("Bound to CPU:", cpu) + numa.bind_to_cpu(cpu) + end end function handlers.n(arg) opts.name = assert(arg) end function handlers.b(arg) opts.bench_file = arg end function handlers.y() opts.hydra = true end function handlers.h() show_usage(0) end + function handlers.reconfigurable() opts.reconfigurable = true end args = lib.dogetopt(args, handlers, "n:hyb:D:", { - help="h", hydra="y", ["bench-file"]="b", duration="D", name="n", cpu=1 }) + help="h", hydra="y", ["bench-file"]="b", duration="D", name="n", cpu=1, + reconfigurable = 0 }) if #args ~= 3 then show_usage(1) end return opts, unpack(args) end @@ -47,8 +55,12 @@ function run(args) if opts.name then engine.claim_name(opts.name) end local graph = config.new() - setup.reconfigurable(setup.load_bench, graph, conf, - inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + if opts.reconfigurable then + setup.reconfigurable(setup.load_bench, graph, conf, + inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + else + setup.load_bench(graph, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + end app.configure(graph) local function start_sampling() From 939f839a87f3a019d3c94b2319305edb78d7da4f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 20:11:51 +0100 Subject: [PATCH 355/631] Better support for features Before, all features were "on". Now you have to declare which features an implementation of a module supports. --- src/lib/yang/schema.lua | 40 +++++++++++++++++++++++++++--------- src/program/lwaftr/setup.lua | 3 +++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 99188d2ea6..e152252b8b 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -590,14 +590,22 @@ local function inherit_config(schema, config) return schema end +local default_features = {} +-- Features should be a table whose keys are module names and whose +-- values are feature name -> boolean tables. +function set_default_features(features) + default_features = features +end + -- Inline "grouping" into "uses". -- Inline "submodule" into "include". -- Inline "imports" into "module". --- Inline "typedef" into "type". (TODO) +-- Inline "typedef" into "type". -- Resolve if-feature. -- Warn on any "when", resolving them as being true. -- Resolve all augment and refine nodes. (TODO) function resolve(schema, features) + if features == nil then features = default_features end local function pop_prop(node, prop) local val = node[prop] node[prop] = nil @@ -687,7 +695,9 @@ function resolve(schema, features) end if node.kind == 'module' or node.kind == 'submodule' then visit_top_level(node, env, 'extensions') - visit_top_level(node, env, 'features') + -- Because features can themselves have if-feature, expand them + -- lazily. + env.features = visit_lazy(pop_prop(node, 'features'), env) visit_top_level(node, env, 'identities') for _,prop in ipairs({'rpcs', 'notifications'}) do node[prop] = shallow_copy(node[prop]) @@ -698,12 +708,21 @@ function resolve(schema, features) if node.input then node.input = visit(node.input, env) end if node.output then node.output = visit(node.output, env) end end + if node.kind == 'feature' then + node.module_id = lookup(env, 'module_id', '_') + end for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do - if not pcall(lookup, env, 'features', feature) then + local feature_node = lookup_lazy(env, 'features', feature) + if node.kind == 'feature' then + -- This is a feature that depends on a feature. These we + -- keep in the environment but if the feature is + -- unavailable, we mark it as such. + local mod, id = feature_node.module_id, feature_node.id + if not (features[mod] or {})[id] then node.unavailable = true end + elseif feature_node.unavailable then return nil, env end end - if node.type then node.type = visit_type(node.type, env) if not node.primitive_type then @@ -745,7 +764,8 @@ function resolve(schema, features) linked[node.id] = 'pending' node = shallow_copy(node) local module_env = {env=env, prefixes={}, extensions={}, features={}, - identities={}, typedefs={}, groupings={}} + identities={}, typedefs={}, groupings={}, + module_id={_=node.id}} node.body = shallow_copy(node.body) node.rpcs = shallow_copy(node.rpcs) node.notifications = shallow_copy(node.notifications) @@ -764,7 +784,7 @@ function resolve(schema, features) end if node.prefix then assert(node.kind == 'module', node.kind) - module_env.prefixes[node.prefix] = node.namespace + module_env.prefixes[node.prefix] = node.id module_env.prefix = {_=node.prefix} end for k,v in pairs(pop_prop(node, 'imports')) do @@ -773,7 +793,7 @@ function resolve(schema, features) -- Is this OK? local schema, env = load_schema_by_name(v.id, v.revision_date) local prefix = v.prefix - module_env.prefixes[prefix] = schema.namespace + module_env.prefixes[prefix] = schema.id for _,prop in ipairs({'extensions', 'features', 'identities', 'typedefs', 'groupings'}) do for k,v in pairs(env[prop]) do @@ -791,8 +811,7 @@ function resolve(schema, features) return node, env end schema = shallow_copy(schema) - return link(schema, {features=(features or {}), - submodules=pop_prop(schema, 'submodules')}) + return link(schema, {submodules=pop_prop(schema, 'submodules')}) end local primitive_types = { @@ -916,7 +935,8 @@ function selftest() -- but not the schema itself. assert(not schema.features) assert(env.features["bowl"]) - assert(env.features["bowl"].description == 'A fruit bowl') + -- Poke through lazy features abstraction by invoking thunk. + assert(env.features["bowl"]().description == 'A fruit bowl') -- Check that groupings get inlined into their uses. assert(schema.body['fruit-bowl']) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index a59ddc0590..f6237087bb 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -18,6 +18,9 @@ local ipv4 = require("lib.protocol.ipv4") local ethernet = require("lib.protocol.ethernet") local ipv4_ntop = require("lib.yang.util").ipv4_ntop +local lwaftr_yang_features = {['ietf-softwire']={binding=true, br=true}} +require('lib.yang.schema').set_default_features(lwaftr_yang_features) + local function convert_ipv4(addr) if addr ~= nil then return ipv4:pton(ipv4_ntop(addr)) end end From 50f9d9f46dcedd0742b2fb6761aa006930cd7398 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 20:40:56 +0100 Subject: [PATCH 356/631] Visit union type members --- src/lib/yang/schema.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index e152252b8b..1050c7b467 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -671,6 +671,13 @@ function resolve(schema, features) else -- If the type name wasn't bound, it must be primitive. assert(primitive_types[node.id], 'unknown type: '..node.id) + if node.id == 'union' then + local union = {} + for _,type in ipairs(node.union) do + table.insert(union, visit_type(type, env)) + end + node.union = union + end node.primitive_type = node.id end return node From 6d2add0ff7c08eeff1bcd4602925305b6a593706 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 20:41:09 +0100 Subject: [PATCH 357/631] Hackily choose first union type member in data grammar In the interests of time, sadly we need to just get going with the ietf-softwire schema, so we're going to just support the first type in a union type. --- src/lib/yang/data.lua | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index e7702499d8..ce15047b9e 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -37,6 +37,13 @@ local function table_string_key(keys) return string_key end +-- We need to properly support unions. It's a big FIXME! As an +-- intermediate step, we pick the first type in the union. Terrible. +local function elide_unions(t) + while t.primitive_type == 'union' do t = t.union[1] end + return t +end + function data_grammar_from_schema(schema) local function struct_ctype(members) local member_names = {} @@ -82,8 +89,9 @@ function data_grammar_from_schema(schema) ctype=struct_ctype(members)}} end handlers['leaf-list'] = function(node) - return {[node.id]={type='array', element_type=node.type, - ctype=value_ctype(node.type)}} + local t = elide_unions(node.type) + return {[node.id]={type='array', element_type=t, + ctype=value_ctype(t)}} end function handlers.list(node) local members=visit_body(node) @@ -99,8 +107,9 @@ function data_grammar_from_schema(schema) end function handlers.leaf(node) local ctype - if node.default or node.mandatory then ctype=value_ctype(node.type) end - return {[node.id]={type='scalar', argument_type=node.type, + local t = elide_unions(node.type) + if node.default or node.mandatory then ctype=value_ctype(t) end + return {[node.id]={type='scalar', argument_type=t, default=node.default, mandatory=node.mandatory, ctype=ctype}} end From 7f0468e2b7cf2560b28fce26ef7076fdb4ce90b6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 2 Dec 2016 20:42:03 +0100 Subject: [PATCH 358/631] Add support for get-config in terms of ietf-softwire --- src/apps/config/support/snabb-softwire-v1.lua | 90 ++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 864f11703d..675d5dc307 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -4,6 +4,7 @@ local ffi = require('ffi') local app = require('core.app') local data = require('lib.yang.data') local yang = require('lib.yang.yang') +local cltable = require('lib.cltable') local path_mod = require('lib.yang.path') local generic = require('apps.config.support').generic_schema_config_support @@ -80,14 +81,101 @@ local function compute_apps_to_restart_after_configuration_update( end end +local function memoize1(f) + local memoized_arg, memoized_result + return function(arg) + if arg == memoized_arg then return memoized_result end + memoized_result = f(arg) + memoized_arg = arg + return memoized_result + end +end + +local ietf_br_instance_grammar +local function get_ietf_br_instance_grammar() + if not ietf_br_instance_grammar then + local schema = yang.load_schema_by_name('ietf-softwire') + local grammar = data.data_grammar_from_schema(schema) + grammar = assert(grammar.members['softwire-config']) + grammar = assert(grammar.members['binding']) + grammar = assert(grammar.members['br']) + grammar = assert(grammar.members['br-instances']) + grammar = assert(grammar.members['br-instance']) + ietf_br_instance_grammar = grammar + end + return ietf_br_instance_grammar +end + +local ietf_softwire_grammar +local function get_ietf_softwire_grammar() + if not ietf_softwire_grammar then + local grammar = get_ietf_br_instance_grammar() + grammar = assert(grammar.values['binding-table']) + grammar = assert(grammar.members['binding-entry']) + ietf_softwire_grammar = grammar + end + return ietf_softwire_grammar +end + +local function cltable_for_grammar(grammar) + assert(grammar.key_ctype) + assert(not grammar.value_ctype) + local key_t = data.typeof(grammar.key_ctype) + return cltable.new({key_type=key_t}), key_t +end + +local function ietf_binding_table_from_native(bt) + local ret, key_t = cltable_for_grammar(get_ietf_softwire_grammar()) + local psid_key_t = data.typeof('struct { uint32_t ipv4; }') + for entry in bt.softwire:iterate() do + local psid_map = bt.psid_map[psid_key_t({ipv4=entry.key.ipv4})] + if psid_map then + local k = key_t({ binding_ipv6info = entry.value.b4_ipv6 }) + local v = { + binding_ipv4_addr = entry.key.ipv4, + port_set = { + psid_offset = psid_map.psid_offset, + psid_len = psid_map.psid_len, + psid = entry.key.psid + }, + br_ipv6_addr = bt.br_address[entry.value.br+1], + } + ret[k] = v + end + end + return ret +end + local function ietf_softwire_translator () local ret = {} function ret.get_config(native_config) - error('unimplemented') + -- Such nesting, very standard, wow + local br_instance, br_instance_key_t = + cltable_for_grammar(get_ietf_br_instance_grammar()) + br_instance[br_instance_key_t({id=1})] = { + -- FIXME + tunnel_payload_mtu = 0, + tunnel_path_mru = 0, + binding_table = { + binding_entry = ietf_binding_table_from_native( + native_config.softwire_config.binding_table) + } + } + return { + softwire_config = { + binding = { + br = { + br_instances = { br_instance = br_instance } + } + } + } + } end + ret.get_config = memoize1(ret.get_config) function ret.get_state(native_state) error('unimplemented') end + ret.get_state = memoize1(ret.get_state) function ret.set_config(native_config, path, data) error('unimplemented') end From 995d3986a28a3480241e7d794fa26845747ede45 Mon Sep 17 00:00:00 2001 From: Christian Graf Date: Sat, 3 Dec 2016 22:19:31 +0100 Subject: [PATCH 359/631] incorporated Diego's comments and simplified the input pcap --- .../snabbvmx/doc/README.troubleshooting.md | 123 +++++++++--------- 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/src/program/snabbvmx/doc/README.troubleshooting.md b/src/program/snabbvmx/doc/README.troubleshooting.md index 4df9b1cee6..91322072d3 100644 --- a/src/program/snabbvmx/doc/README.troubleshooting.md +++ b/src/program/snabbvmx/doc/README.troubleshooting.md @@ -169,7 +169,7 @@ everything is working as expected. **preparation** - have tcpdump, snabb or vmxlwaftr installed. -- should work on xeon or i7; will fail on Celeron chips +- should work on xeon or i7; will fail on Celeron chips - hugepages : Ensure hugepages is set in sysctl "vm.nr_hugepages = 5" in sysctl.conf If using vmxlwaftr, then this is done by the scripts automatically @@ -195,7 +195,7 @@ The tests can also be run when executing **make test** (only if the Snabb source is available). -### make tests +### Make tests ```bash $ sudo make test @@ -320,15 +320,15 @@ NOTE: Currently the test is not working correctly: the returned MAC should be `02:99:99:99:99:99`. -## crafting and running end-to-end tests +## Crafting and running end-to-end tests -### brief steps +### Brief steps -(1) run the interactive check with "-r" parm to derive the counters and out.pcaps +(1) run the interactive check with "-r" param to derive the counters and out.pcaps -`snabb/src/snabb snabbvmx check -r ./CONF.cfg "./V4-IN.PCAP" "./V6-IN.PCAP" "./outv4.pcap" "./outv6.pcap" COUNTERS.lua` +`snabb snabbvmx check -r ./CONF.cfg "./V4-IN.PCAP" "./V6-IN.PCAP" "./outv4.pcap" "./outv6.pcap" COUNTERS.lua` -(2) place derived counters.lua in "snabb/src/program/snabbvmx/tests/end-to-end/data/counters" +(2) place derived counters.lua in "snabb/src/program/snabbvmx/tests/end-to-end/data/counters" (3) place derived and expected out.pcaps in "snabb/src/program/snabbvmx/tests/end-to-end/data" @@ -341,7 +341,7 @@ NOTE: Currently the test is not working correctly: the returned MAC should be ### How the system test works in detail -The way the test system works is by passing the input IPv4/IPv6 packets (via pre-recorded pcap-files) to the lwAFTR and comparing the expected packet output (pcap-file) to the packet output that the lwAFTR has generated. +The way the test system works is by passing the input IPv4/IPv6 packets (via pre-recorded pcap-files) to the lwAFTR and comparing the expected packet output (pcap-file) to the packet output that the lwAFTR has generated. The optional counters file is compared too to the actual counters file obtained after running the test. This is beeing reflected in snabbvmx check syntax: `CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP [COUNTERS.LUA]` @@ -350,7 +350,7 @@ V4-OUT.PCAP, V6-OUT.PCAP and COUNTERS.LUA are expected output. These output is c **lwaftr vs snabbvmx** -As both lwaftr and snabbvmx provide a different functionality and use different config-files, both the lwaftr and snabbvmx have their dedicated end-to-end tests. +As both lwaftr and snabbvmx provide a different functionality and use different config-files, both the lwaftr and snabbvmx have their dedicated end-to-end tests. Although SnabbVMX works on a single interface, `snabbvmx check` requires that the packet split (IPv4 / IPv6) is already done and provides a split output too. @@ -370,13 +370,13 @@ in Snabb's lwAFTR code, resulting in the addition of a new test to the lwAFTR's test suite. -**lwaftr** +**lwaftr** ``` cd src/program/lwaftr/tests/end-to-end ``` -**snabbvmx** +**snabbvmx** ``` cd src/program/snabbvmx/tests/end-to-end @@ -400,12 +400,12 @@ Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP Parameters: -- CONF: SnabbVMX (icmp_snabbvmx-lwaftr-xe.cfg) or lwaftr (icmp_snabbvmx-lwaftr-xe1.conf) configuration file. -- V4-IN.PCAP : Incoming IPv4 packets (from Internet). -- V6-IN.PCAP : Incoming IPv6 packets (from b4). -- V4-OUT.PCAP : Outgoing IPv4 packets (to Internet, decapsulated). -- V6-OUT.PCAP : Outgoing IPv6 packets (to b4, encapsulated) -- [COUNTERS.LUA] : Lua file with counter values. Will be regenerated via [-r] parm +- **CONF** : SnabbVMX (icmp_snabbvmx-lwaftr-xe.cfg) or lwaftr (icmp_snabbvmx-lwaftr-xe1.conf) configuration file. +- **V4-IN.PCAP** : Incoming IPv4 packets (from Internet). +- **V6-IN.PCAP** : Incoming IPv6 packets (from b4). +- **V4-OUT.PCAP** : Outgoing IPv4 packets (to Internet, decapsulated). +- **V6-OUT.PCAP** : Outgoing IPv6 packets (to b4, encapsulated) +- **[COUNTERS.LUA]** : Lua file with counter values. Will be regenerated via [-r] param ## How to run SnabbVMX interactive end-to-end test @@ -416,9 +416,9 @@ configuration and binding table. With that information and knowing the error report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get decapsulated, etc), you craft a hand-made packet that meets the testing case. -**obtaining the config-files** - -To run a test, the following config-files are required: +**obtaining the config-files** + +To run a test, the following config-files are required: - the binding-table : binding_table.txt.s - lwaftr conf : snabbvmx-lwaftr-xe[0-9].conf @@ -427,13 +427,13 @@ To run a test, the following config-files are required: If you are running lwaftr check, then snabbvmx config-file (snabbvmx-lwaftr-xe[0-9].cfg) is not required It is fine to copy or manually craft the config-files. -A running snabbvmx can be used as well to copy the config-files from the running container +A running snabbvmx can be used as well to copy the config-files from the running container To gain the used config-files from the running container, either run the collect-support-infos.sh (https://github.com/mwiget/vmxlwaftr/blob/igalia/SUPPORT-INFO.md) or execute a shell within the dockers container and copy configs and binding-table from the /tmp directory. Note: the check application is just using a single interface. If the running container consists of two or more snabb-instances, then just take one of them for when running the check. - -**collect-support-infos.sh** + +**collect-support-infos.sh** ``` lab@ubuntu1:~/vmxlwaftr/tests$ ./collect-support-infos.sh lwaftr3-16.2R3 @@ -474,7 +474,7 @@ lab@ubuntu1:~/vmxlwaftr/tests/t1$ tar -tvzf support-info-20161108-1335.tgz -rwxr-xr-x root/root 2707019 2016-10-31 14:06 usr/local/bin/snabb ``` -**config-files within /tmp inside docker container** +**config-files within /tmp inside docker container** The snabbvmx config-files can be derived from the container's shell as well directly within the /tmp directory @@ -491,11 +491,11 @@ binding_table.txt.s.o junos-vmx-x86-64-16.1-20160926.0.qcow2 pci_xe1 snabbv Note: press ctrl p ctrl q to exit the containers shell -**some adoption of config-files is required** +**some adoption of config-files is required** The advantage of snabbvmx is the dynamic next-hop resolution via Junos. When running the the lwaftr or snabbvmx app standalone, then the next-hop resolution via Junos is missing. The config-files are required to get modified for static next-hop configuration. -**config as derived from a running vmxlwaftr container** +**config as derived from a running vmxlwaftr container** ``` lab@ubuntu1:~/vmxlwaftr/tests/t1$ cat snabbvmx-lwaftr-xe1.cfg @@ -516,19 +516,20 @@ return { } ``` -**Adopted static next-hop configuration to use with vmxlwaftr** +**Adopted static next-hop configuration to use with vmxlwaftr** To change configuration for static next-hop, below changes are required: - + - cache_refresh_interval = 0 (turns off next-hop learning via Junos) - mac_address (thats the own/self mac-address fo lwaftr) - next_hop_mac (next-hop mac to send the packets to) -Note: The snabbvmx config icmp_snabbvmx-lwaftr-xe1.cfg references the snabb-configuration file icmp_snabbvmx-lwaftr-xe1.conf via the "lwaftr" directive. +Note: The snabbvmx config icmp_snabbvmx-lwaftr-xe1.cfg references the snabb-configuration file icmp_snabbvmx-lwaftr-xe1.conf via the "lwaftr" directive. When running "snabbvmx check" then both the lwaftr and the snabbvmx config-files must be provided. ``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat icmp_snabbvmx-lwaftr-xe1.cfg +cd snabb/src/program/snabbvmx/tests/end-to-end/data$ +cat icmp_snabbvmx-lwaftr-xe1.cfg return { lwaftr = "icmp_snabbvmx-lwaftr-xe1.conf", settings = { @@ -548,37 +549,37 @@ return { next_hop_mac = "90:e2:ba:94:2a:bc", <<< the next-hop mac to use for outgoing packets }, } -``` +``` -**The input pcaps** +**The input pcaps** The snabb "check" app requires one or two input pcaps. -It is ok to: +It is ok to: - only feed V4-IN.PCAP - only feed the V6-IN.PCAP - or feed both V4-IN.PCAP and V6-IN.PCAP -Note for interactive tests: -When only feeding one pcap, then the other empty pcap must be the "empty.pcap" in +Note for interactive tests:
+When only feeding one pcap as input, then the other empty pcap must be the "empty.pcap" in "src/program/snabbvmx/tests/end-to-end/data/empty.pcap" and not an empty string like "" ``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ ls empty.pcap -empty.pcap -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ file empty.pcap +cd snabb/src/program/snabbvmx/tests/end-to-end/data$ +$ file empty.pcap empty.pcap: tcpdump capture file (little-endian) - version 2.4 (Ethernet, capture length 65535) -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -r empty.pcap +$ tcpdump -r empty.pcap reading from file empty.pcap, link-type EN10MB (Ethernet) ``` -**sample icmp-ipv4-in.pcap** +**sample icmp-ipv4-in.pcap** For any input pcap it makes sense to keep it short - ideally a single packet to check correctness of the lwaftr. The input packet "11:26:07.168372 IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain]" is matching the binding-table for PSID=1. ``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat cg-binding_table.txt.s +cd snabb/src/program/snabbvmx/tests/end-to-end/data$ +$ cat cg-binding_table.txt.s psid_map { 10.10.0.0 {psid_length=6, shift=10} 10.10.0.1 {psid_length=6, shift=10} @@ -599,13 +600,11 @@ reading from file ipv4-in.pcap, link-type EN10MB (Ethernet) 11:26:07.168372 IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] ``` +**running the interactive check** - -**running the interactive check** - -The interactive check is performed via `snabb snabbvmx check -r`. Using the "-r" parm instructs the check to generate the counters and out.pcap files. +The interactive check is performed via `snabb snabbvmx check -r`. Using the "-r" param instructs the check to generate the counters and out.pcap files. As there is no IPv6 input, the "empty.pcap" is configured. - + ``` sudo ~/vmx-docker-lwaftr/snabb/src/snabb snabbvmx check -r ./snabbvmx-lwaftr-xe1.cfg "./ipv4-in.pcap" "./empty.pcap" "./outv4.pcap" "./outv6.pcap" test.lua loading compiled binding table from ./binding_table.txt.s.o @@ -620,34 +619,35 @@ done **results** The -r flag is set, as such the resulting counters file test.lua and the out.pcap files are generated freshly. -The results below show a correct processing of the lwatr: +The results below show a correct processing of the lwatr: - the counters-file lists one IPv4 packet as input - as the input-packet matches the binding-table, one IPv6 packet output - outv4.pcap is empty -- outv6.pcap shows the resulting encapsulated lw4o6 packet +- outv6.pcap shows the resulting encapsulated lw4o6 packet ``` -lab@ubuntu1:~/vmx-docker-lwaftr/snabb/src/program/snabbvmx/tests/end-to-end/data$ cat test.lua +$ cd snabb/src/program/snabbvmx/tests/end-to-end/data +$ cat test.lua return { ["in-ipv4-bytes"] = 42, ["in-ipv4-packets"] = 1, ["out-ipv6-bytes"] = 82, ["out-ipv6-packets"] = 1, } -lab@ubuntu1:~/vmx-docker-lwaftr/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv4.pcap +$ tcpdump -n -r outv4.pcap reading from file outv4.pcap, link-type EN10MB (Ethernet) -lab@ubuntu1:~/vmx-docker-lwaftr/snabb/src/program/snabbvmx/tests/end-to-end/data$ tcpdump -n -r outv6.pcap +$ tcpdump -n -r outv6.pcap reading from file outv6.pcap, link-type EN10MB (Ethernet) 01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::400: IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] ``` -**summary interactive check** +**summary interactive check** At this stage the interactive test is finished. -The following is defined: +The following is defined: - lwaftr and snabbvmx configs - binding-table @@ -662,15 +662,15 @@ Tip: packets always arrive only in one interface, but the output might be empty or non-empty for both IPv4 and IPv6. -## adding the sample-test towards the scripted end-to-end tests +## Adding the sample-test towards the scripted end-to-end tests -**in short** +### In short - place counter file and out.pcaps into correct directory - edit "test_env.sh" and add the test - run the test -**details** +### Detailed steps **directory-structure** place out.pcap into the data-drectory @@ -695,17 +695,17 @@ configuration files and binding tables. * **end-to-end-vlan.sh**: Runs **core-end-to-end.sh** on VLAN packets. * **selftest.sh**: Runs both **end-to-end.sh** and **end-to-end-vlan.sh** -` **adding the sample test**: All tests are defined in the "test_env.sh" file. -This file has to be edited to include the sample test -The check app always checks the provided data to decide if the test is successful or not: +This "test_env.sh" file has to be edited to include the sample test. +The check app always checks the provided data to decide if the test is successful or not:
+ - if the optional counters-file is provided, then it must match - the resulting out.pcap files must always match -**pass-criteria without the optional counters.lua file** +**pass-criteria without the optional counters.lua file** ``` src/program/snabbvmx/tests/end-to-end$ vi test_env.sh @@ -754,7 +754,8 @@ All end-to-end lwAFTR tests passed. If the counters file shall be taken into consideration as well (e.g. to count dropped frames), then just one line needs to be changed from "" to the counters file "test.lua" ``` -lab@ubuntu1:~/latest-snabb-binary/snabb/src/program/snabbvmx/tests/end-to-end$ vi test_env.sh +$ cd snabb/src/program/snabbvmx/tests/end-to-end +$ vi test_env.sh ... "sample test" "snabbvmx-lwaftr-xe1.cfg" "ipv4-in.pcap" "" "" "outv6.pcap" From d05ae2535b531a04c0270a9301503654d78c258b Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Sun, 4 Dec 2016 13:02:50 +0000 Subject: [PATCH 360/631] Added front-end lwaftr tests --- src/program/lwaftr/selftest.sh | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 src/program/lwaftr/selftest.sh diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh new file mode 100755 index 0000000000..4498f050f6 --- /dev/null +++ b/src/program/lwaftr/selftest.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e # Exit on any errors + +SKIPPED_CODE=43 + +if [[ $EUID != 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +# These are tests for lwaftr front ends. + +echo "Testing snabb lwaftr bench" +../../snabb lwaftr bench -D 0.1 tests/data/icmp_on_fail.conf \ + tests/data/tcp-frominet-bound.pcap tests/data/tcp-fromb4-ipv6.pcap + +echo "Testing snabb lwaftr bench --reconfigurable" +../../snabb lwaftr bench --reconfigurable -D 0.1 tests/data/icmp_on_fail.conf \ + tests/data/tcp-frominet-bound.pcap tests/data/tcp-fromb4-ipv6.pcap + +# The rest of the tests require real hardware + +if [ -z "$SNABB_PCI0" ]; then + echo "Skipping tests which require real hardware, SNABB_PCI0 not set" + exit $SKIPPED_CODE +fi + +echo "Testing snabb lwaftr run" +sudo ../../snabb lwaftr run -D 0.1 --conf tests/data/icmp_on_fail.conf \ + --on-a-stick "$SNABB_PCI0" + +echo "Testing snabb lwaftr run --reconfigurable" +sudo ../../snabb lwaftr run -D 0.1 --reconfigurable \ + --conf tests/data/icmp_on_fail.conf --on-a-stick "$SNABB_PCI0" From 764c97162724e4d0c09508533fc348c5b8717cf6 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Sun, 4 Dec 2016 22:00:31 +0100 Subject: [PATCH 361/631] Make lwaftr run --cpu work with multiprocessing --- src/program/lwaftr/run/run.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index ba322e87b3..9ebe9ea3bf 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -43,6 +43,16 @@ function parse_args(args) if not cpu or cpu ~= math.floor(cpu) or cpu < 0 then fatal("Invalid cpu number: "..arg) end + + if opts.reconfigurable then + S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) + local wanted_node = numa.cpu_get_numa_node(cpu) + numa.bind_to_numa_node(wanted_node) + print("Bound to numa node:", wanted_node) + else + print("Bound to CPU:", cpu) + numa.bind_to_cpu(cpu) + end end handlers['real-time'] = function(arg) if not S.sched_setscheduler(0, "fifo", 1) then From 73cd0c5ed87ab20808a3823054f24a12a35e7dd5 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Sun, 4 Dec 2016 22:09:19 +0100 Subject: [PATCH 362/631] Fixed selftest paths to run from make test --- src/program/lwaftr/selftest.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh index 4498f050f6..553a24d824 100755 --- a/src/program/lwaftr/selftest.sh +++ b/src/program/lwaftr/selftest.sh @@ -3,6 +3,7 @@ set -e # Exit on any errors SKIPPED_CODE=43 +TDIR="program/lwaftr/tests/data/" if [[ $EUID != 0 ]]; then echo "This script must be run as root" @@ -12,12 +13,12 @@ fi # These are tests for lwaftr front ends. echo "Testing snabb lwaftr bench" -../../snabb lwaftr bench -D 0.1 tests/data/icmp_on_fail.conf \ - tests/data/tcp-frominet-bound.pcap tests/data/tcp-fromb4-ipv6.pcap +./snabb lwaftr bench -D 0.1 ${TDIR}/icmp_on_fail.conf \ + ${TDIR}/tcp-frominet-bound.pcap ${TDIR}/tcp-fromb4-ipv6.pcap echo "Testing snabb lwaftr bench --reconfigurable" -../../snabb lwaftr bench --reconfigurable -D 0.1 tests/data/icmp_on_fail.conf \ - tests/data/tcp-frominet-bound.pcap tests/data/tcp-fromb4-ipv6.pcap +./snabb lwaftr bench --reconfigurable -D 0.1 ${TDIR}/icmp_on_fail.conf \ + ${TDIR}/tcp-frominet-bound.pcap ${TDIR}/tcp-fromb4-ipv6.pcap # The rest of the tests require real hardware @@ -27,9 +28,9 @@ if [ -z "$SNABB_PCI0" ]; then fi echo "Testing snabb lwaftr run" -sudo ../../snabb lwaftr run -D 0.1 --conf tests/data/icmp_on_fail.conf \ +sudo ./snabb lwaftr run -D 0.1 --conf ${TDIR}/icmp_on_fail.conf \ --on-a-stick "$SNABB_PCI0" echo "Testing snabb lwaftr run --reconfigurable" -sudo ../../snabb lwaftr run -D 0.1 --reconfigurable \ - --conf tests/data/icmp_on_fail.conf --on-a-stick "$SNABB_PCI0" +sudo ./snabb lwaftr run -D 0.1 --reconfigurable \ + --conf ${TDIR}/icmp_on_fail.conf --on-a-stick "$SNABB_PCI0" From 79c70737d1ca541e3a9eb8081f7aeea8d930ae1a Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Sun, 4 Dec 2016 22:42:14 +0100 Subject: [PATCH 363/631] Added an automated test for snabb config get --- src/program/lwaftr/selftest.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh index 553a24d824..b3f292a7fa 100755 --- a/src/program/lwaftr/selftest.sh +++ b/src/program/lwaftr/selftest.sh @@ -17,8 +17,13 @@ echo "Testing snabb lwaftr bench" ${TDIR}/tcp-frominet-bound.pcap ${TDIR}/tcp-fromb4-ipv6.pcap echo "Testing snabb lwaftr bench --reconfigurable" -./snabb lwaftr bench --reconfigurable -D 0.1 ${TDIR}/icmp_on_fail.conf \ - ${TDIR}/tcp-frominet-bound.pcap ${TDIR}/tcp-fromb4-ipv6.pcap +./snabb lwaftr bench --reconfigurable -D 1 ${TDIR}/icmp_on_fail.conf \ + ${TDIR}/tcp-frominet-bound.pcap ${TDIR}/tcp-fromb4-ipv6.pcap & + +LEADER_PID=$! + +./snabb config get $LEADER_PID / + # The rest of the tests require real hardware From 7d8710bb243e0ab2e50f84687077deba895dea91 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 10:44:05 +0100 Subject: [PATCH 364/631] Initial get-state support over ietf-softwire Full support is waiting on incorporation of more counters into the yang module. --- src/apps/config/support/snabb-softwire-v1.lua | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 675d5dc307..c9f711b515 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -173,7 +173,28 @@ local function ietf_softwire_translator () end ret.get_config = memoize1(ret.get_config) function ret.get_state(native_state) - error('unimplemented') + -- Even though this is a different br-instance node, it is a + -- cltable with the same key type, so just re-use the key here. + local br_instance, br_instance_key_t = + cltable_for_grammar(get_ietf_br_instance_grammar()) + br_instance[br_instance_key_t({id=1})] = { + -- FIXME! + sentPacket = 0, + sentByte = 0, + rcvdPacket = 0, + rcvdByte = 0, + droppedPacket = 0, + droppedByte = 0 + } + return { + softwire_state = { + binding = { + br = { + br_instances = { br_instance = br_instance } + } + } + } + } end ret.get_state = memoize1(ret.get_state) function ret.set_config(native_config, path, data) From 4e1c534f941f4397a962450b458f9fed152381f7 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 5 Dec 2016 12:42:46 +0100 Subject: [PATCH 365/631] Add missing lwaftr counters to snabb-softwire-v1 This adds a number of missing counters, usually the -bytes or -packets counterparts (one was usually missing). It also sorts them alphabetically in the file. --- src/lib/yang/snabb-softwire-v1.yang | 427 ++++++++++++++++------------ 1 file changed, 251 insertions(+), 176 deletions(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 27e49b9483..bdf9a07029 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -367,297 +367,372 @@ module snabb-softwire-v1 { } } } - + container softwire-state { description "State data about lwaftr."; config false; - - /* Decapsulation B4 queue */ - leaf in-ipv6-packets { + + leaf drop-all-ipv4-iface-bytes { type yang:zero-based-counter64; - description "All valid outgoing IPv4 packets."; + description + "All dropped packets and bytes that came in over IPv4 interfaces, + whether or not they actually IPv4 (they only include data about + packets that go in/out over the wires, excluding internally generated + ICMP packets)."; } - - leaf drop-misplaced-not-ipv6-bytes { + leaf drop-all-ipv4-iface-packets { type yang:zero-based-counter64; - description "Non-IPv6 packets incoming on IPv6 link."; + description + "All dropped packets and bytes that came in over IPv4 interfaces, + whether or not they actually IPv4 (they only include data about + packets that go in/out over the wires, excluding internally generated + ICMP packets)."; } - - leaf drop-unknown-protocol-ipv6-bytes { + leaf drop-all-ipv6-iface-bytes { type yang:zero-based-counter64; - description "Packets with an unknown IPv6 protocol."; + description + "All dropped packets and bytes that came in over IPv6 interfaces, + whether or not they actually IPv6 (they only include data about packets + that go in/out over the wires, excluding internally generated ICMP + packets)."; + } + leaf drop-all-ipv6-iface-packets { + type yang:zero-based-counter64; + description + "All dropped packets and bytes that came in over IPv6 interfaces, + whether or not they actually IPv6 (they only include data about packets + that go in/out over the wires, excluding internally generated ICMP + packets)."; + } + leaf drop-bad-checksum-icmpv4-bytes { + type yang:zero-based-counter64; + description "ICMPv4 packets dropped because of a bad checksum."; + } + leaf drop-bad-checksum-icmpv4-packets { + type yang:zero-based-counter64; + description "ICMPv4 packets dropped because of a bad checksum."; + } + leaf drop-in-by-policy-icmpv4-bytes { + type yang:zero-based-counter64; + description "Incoming ICMPv4 packets dropped because of current policy."; + } + leaf drop-in-by-policy-icmpv4-packets { + type yang:zero-based-counter64; + description "Incoming ICMPv4 packets dropped because of current policy."; } - leaf drop-in-by-policy-icmpv6-bytes { type yang:zero-based-counter64; description "Incoming ICMPv6 packets dropped because of current policy."; } - - leaf out-icmpv4-packets { + leaf drop-in-by-policy-icmpv6-packets { type yang:zero-based-counter64; - description "Internally generated ICMPv4 packets."; + description "Incoming ICMPv6 packets dropped because of current policy."; } - - leaf drop-too-big-type-but-not-code-icmpv6-bytes { + leaf drop-in-by-rfc7596-icmpv4-bytes { type yang:zero-based-counter64; - description - "Packet's ICMP type was 'Packet too big' but its ICMP code was not an - acceptable one for this type."; + description + "Incoming ICMPv4 packets with no destination (RFC 7596 section 8.1)."; } - - leaf drop-over-time-but-not-hop-limit-icmpv6-bytes { + leaf drop-in-by-rfc7596-icmpv4-packets { type yang:zero-based-counter64; - description - "Packet's time limit was exceeded, but the hop limit was not."; + description + "Incoming ICMPv4 packets with no destination (RFC 7596 section 8.1)."; + } + leaf drop-ipv4-frag-disabled { + type yang:zero-based-counter64; + description + "If fragmentation is disabled, the only potentially non-zero IPv4 + fragmentation counter is drop-ipv4-frag-disabled. If fragmentation is + enabled, it will always be zero."; + } + leaf drop-ipv4-frag-invalid-reassembly { + type yang:zero-based-counter64; + description + "Two or more IPv4 fragments were received, and reassembly was started, + but was invalid and dropped. Causes include multiple fragments claiming + they are the last fragment, overlapping fragment offsets, or the packet + was being reassembled from too many fragments (the setting is + max_fragments_per_reassembly_packet, and the default is that no packet + should be reassembled from more than 40)."; + } + leaf drop-ipv4-frag-random-evicted { + type yang:zero-based-counter64; + description + "Reassembling an IPv4 packet from fragments was in progress, but the + configured amount of packets to reassemble at once was exceeded, so one + was dropped at random. Consider increasing the setting + max_ipv4_reassembly_packets."; + } + leaf drop-ipv6-frag-disabled { + type yang:zero-based-counter64; + description + "If fragmentation is disabled, the only potentially non-zero IPv6 + fragmentation counter is drop-ipv6-frag-disabled. If fragmentation is + enabled, it will always be zero."; + } + leaf drop-ipv6-frag-invalid-reassembly { + type yang:zero-based-counter64; + description + "Two or more IPv6 fragments were received, and reassembly was started, + but was invalid and dropped. Causes include multiple fragments claiming + they are the last fragment, overlapping fragment offsets, or the packet + was being reassembled from too many fragments (the setting is + max_fragments_per_reassembly_packet, and the default is that no packet + should be reassembled from more than 40)."; + } + leaf drop-ipv6-frag-random-evicted { + type yang:zero-based-counter64; + description + "Reassembling an IPv6 packet from fragments was in progress, but the + configured amount of packets to reassemble at once was exceeded, so one + was dropped at random. Consider increasing the setting + max_ipv6_reassembly_packets."; + } + leaf drop-misplaced-not-ipv4-bytes { + type yang:zero-based-counter64; + description "Non-IPv4 packets incoming on the IPv4 link."; + } + leaf drop-misplaced-not-ipv4-packets { + type yang:zero-based-counter64; + description "Non-IPv4 packets incoming on the IPv4 link."; + } + leaf drop-misplaced-not-ipv6-bytes { + type yang:zero-based-counter64; + description "Non-IPv6 packets incoming on IPv6 link."; + } + leaf drop-misplaced-not-ipv6-packets { + type yang:zero-based-counter64; + description "Non-IPv6 packets incoming on IPv6 link."; + } + leaf drop-no-dest-softwire-ipv4-bytes { + type yang:zero-based-counter64; + description + "No matching destination softwire in the binding table; incremented + whether or not the reason was RFC7596."; + } + leaf drop-no-dest-softwire-ipv4-packets { + type yang:zero-based-counter64; + description + "No matching destination softwire in the binding table; incremented + whether or not the reason was RFC7596."; } - - /* Decapsulation internet queue */ leaf drop-no-source-softwire-ipv6-bytes { type yang:zero-based-counter64; - description + description "No matching source softwire in the binding table; incremented whether or not the reason was RFC7596."; } - - leaf out-ipv4-packets { + leaf drop-no-source-softwire-ipv6-packets { type yang:zero-based-counter64; - description "Valid outgoing IPv4 packets."; + description + "No matching source softwire in the binding table; incremented whether + or not the reason was RFC7596."; } - - leaf hairpin-ipv4-packets { + leaf drop-out-by-policy-icmpv4-packets { type yang:zero-based-counter64; - description "IPv4 packets going to a known b4 (hairpinned)."; + description + "Internally generated ICMPv4 error packets dropped because of current + policy."; } - - leaf drop-out-by-policy-icmpv6-bytes { + leaf drop-out-by-policy-icmpv6-packets { type yang:zero-based-counter64; description "Internally generated ICMPv6 packets dropped because of current policy."; } - + leaf drop-over-mtu-but-dont-fragment-ipv4-bytes { + type yang:zero-based-counter64; + description + "IPv4 packets whose size exceeded the MTU, but the DF (Don't Fragment) + flag was set."; + } + leaf drop-over-mtu-but-dont-fragment-ipv4-packets { + type yang:zero-based-counter64; + description + "IPv4 packets whose size exceeded the MTU, but the DF (Don't Fragment) + flag was set."; + } leaf drop-over-rate-limit-icmpv6-bytes { type yang:zero-based-counter64; - description + description "Packets dropped because the outgoing ICMPv6 rate limit was reached."; } - - leaf out-icmpv6-packets { + leaf drop-over-rate-limit-icmpv6-packets { type yang:zero-based-counter64; - description "Internally generted ICMPv6 error packets."; + description + "Packets dropped because the outgoing ICMPv6 rate limit was reached."; } - - /* Encapsulation internet queue */ - leaf in-ipv4-packets { + leaf drop-over-time-but-not-hop-limit-icmpv6-bytes { type yang:zero-based-counter64; - description "All valid outgoing IPv4 packets."; + description + "Packet's time limit was exceeded, but the hop limit was not."; } - - leaf drop-in-by-policy-icmpv4-bytes { + leaf drop-over-time-but-not-hop-limit-icmpv6-packets { type yang:zero-based-counter64; - description "Incoming ICMPv4 packets dropped because of current policy."; + description + "Packet's time limit was exceeded, but the hop limit was not."; } - - leaf drop-misplaced-not-ipv4-bytes { + leaf drop-too-big-type-but-not-code-icmpv6-bytes { type yang:zero-based-counter64; - description "Non-IPv4 packets incoming on the IPv4 link."; + description + "Packet's ICMP type was 'Packet too big' but its ICMP code was not an + acceptable one for this type."; } - - leaf drop-bad-checksum-icmpv4-bytes { + leaf drop-too-big-type-but-not-code-icmpv6-packets { type yang:zero-based-counter64; - description "ICMPv4 packets dropped because of a bad checksum."; + description + "Packet's ICMP type was 'Packet too big' but its ICMP code was not an + acceptable one for this type."; } - - leaf drop-all-ipv4-iface-bytes { + leaf drop-ttl-zero-ipv4-bytes { type yang:zero-based-counter64; - description - "All dropped packets and bytes that came in over IPv4 interfaces, - whether or not they actually IPv4 (they only include data about - packets that go in/out over the wires, excluding internally generated - ICMP packets)."; + description "IPv4 packets dropped because their TTL was zero."; } - - leaf drop-all-ipv6-iface-bytes { + leaf drop-ttl-zero-ipv4-packets { type yang:zero-based-counter64; - description - "All dropped packets and bytes that came in over IPv6 interfaces, - whether or not they actually IPv6 (they only include data about packets - that go in/out over the wires, excluding internally generated ICMP - packets)."; + description "IPv4 packets dropped because their TTL was zero."; } - - /* Encapsulation B4 queue */ - leaf out-ipv6-packets { + leaf drop-unknown-protocol-icmpv6-bytes { type yang:zero-based-counter64; - description "All valid outgoing IPv6 packets."; + description "Packets with an unknown ICMPv6 protocol."; } - - leaf drop-over-mtu-but-dont-fragment-ipv4-bytes { + leaf drop-unknown-protocol-icmpv6-packets { type yang:zero-based-counter64; - description - "IPv4 packets whose size exceeded the MTU, but the DF (Don't Fragment) - flag was set."; + description "Packets with an unknown ICMPv6 protocol."; } - - leaf drop-ttl-zero-ipv4-bytes { + leaf drop-unknown-protocol-ipv6-bytes { type yang:zero-based-counter64; - description "IPv4 packets dropped because their TTL was zero."; + description "Packets with an unknown IPv6 protocol."; } - - leaf drop-out-by-policy-icmpv4-bytes { + leaf drop-unknown-protocol-ipv6-packets { type yang:zero-based-counter64; - description - "Internally generated ICMPv4 error packets dropped because of current - policy."; + description "Packets with an unknown IPv6 protocol."; } - - leaf drop-no-dest-softwire-ipv4-bytes { + leaf hairpin-ipv4-bytes { type yang:zero-based-counter64; - description - "No matching destination softwire in the binding table; incremented - whether or not the reason was RFC7596."; + description "IPv4 packets going to a known b4 (hairpinned)."; } - - leaf drop-in-by-rfc7596-icmpv4-bytes { + leaf hairpin-ipv4-packets { type yang:zero-based-counter64; - description - "Incoming ICMPv4 packets with no destination (RFC 7596 section 8.1)."; + description "IPv4 packets going to a known b4 (hairpinned)."; } - - /* Fragmentation counters IPv4 */ - leaf drop-ipv4-frag-disabled { + leaf in-ipv4-bytes { type yang:zero-based-counter64; - description - "If fragmentation is disabled, the only potentially non-zero IPv4 - fragmentation counter is drop-ipv4-frag-disabled. If fragmentation is - enabled, it will always be zero."; + description "All valid outgoing IPv4 packets."; } - leaf in-ipv4-frag-needs-reassembly { type yang:zero-based-counter64; description "An IPv4 fragment was received."; } - leaf in-ipv4-frag-reassembled { type yang:zero-based-counter64; description "A packet was successfully reassembled from IPv4 fragments."; } - leaf in-ipv4-frag-reassembly-unneeded { type yang:zero-based-counter64; - description + description "An IPv4 packet which was not a fragment was received - consequently, it did not need to be reassembled. This should be the usual case."; } - - leaf drop-ipv4-frag-invalid-reassembly { + leaf in-ipv4-packets { type yang:zero-based-counter64; - description - "Two or more IPv4 fragments were received, and reassembly was started, - but was invalid and dropped. Causes include multiple fragments claiming - they are the last fragment, overlapping fragment offsets, or the packet - was being reassembled from too many fragments (the setting is - max_fragments_per_reassembly_packet, and the default is that no packet - should be reassembled from more than 40)."; + description "All valid outgoing IPv4 packets."; } - - leaf drop-ipv4-frag-random-evicted { + leaf in-ipv6-bytes { type yang:zero-based-counter64; - description - "Reassembling an IPv4 packet from fragments was in progress, but the - configured amount of packets to reassemble at once was exceeded, so one - was dropped at random. Consider increasing the setting - max_ipv4_reassembly_packets."; + description "All valid outgoing IPv4 packets."; } - - leaf out-ipv4-frag { + leaf in-ipv6-frag-needs-reassembly { type yang:zero-based-counter64; - description - "An outgoing packet exceeded the configured IPv4 MTU, so needed to be - fragmented. This may happen, but should be unusual."; + description "An IPv6 fragment was received."; } - - leaf out-ipv4-frag-not { + leaf in-ipv6-frag-reassembled { type yang:zero-based-counter64; - description - "An outgoing packet was small enough to pass through unfragmented - this - should be the usual case."; + description "A packet was successfully reassembled from IPv6 fragments."; + } + leaf in-ipv6-frag-reassembly-unneeded { + type yang:zero-based-counter64; + description + "An IPv6 packet which was not a fragment was received - consequently, it + did not need to be reassembled. This should be the usual case."; + } + leaf in-ipv6-packets { + type yang:zero-based-counter64; + description "All valid outgoing IPv4 packets."; + } + leaf ingress-packet-drops { + type yang:zero-based-counter64; + description "Packets dropped due to ingress filters."; } - leaf memuse-ipv4-frag-reassembly-buffer { type yang:zero-based-counter64; - description + description "The amount of memory being used by the statically sized data structure for reassembling IPv4 fragments. This is directly proportional to the setting max_ipv4_reassembly_packets."; } - - /* Fragmentation counters IPv6 */ - leaf drop-ipv6-frag-disabled { + leaf memuse-ipv6-frag-reassembly-buffer { type yang:zero-based-counter64; - description - "If fragmentation is disabled, the only potentially non-zero IPv6 - fragmentation counter is drop-ipv6-frag-disabled. If fragmentation is - enabled, it will always be zero."; + description + "The amount of memory being used by the statically sized data structure + for reassembling IPv6 fragments. This is directly proportional to the + setting max_ipv6_reassembly_packets."; } - - leaf in-ipv6-frag-needs-reassembly { + leaf out-icmpv4-bytes { type yang:zero-based-counter64; - description "An IPv6 fragment was received."; + description "Internally generated ICMPv4 packets."; } - - leaf in-ipv6-frag-reassembled { + leaf out-icmpv4-packets { type yang:zero-based-counter64; - description "A packet was successfully reassembled from IPv6 fragments."; + description "Internally generated ICMPv4 packets."; } - - leaf in-ipv6-frag-reassembly-unneeded { + leaf out-icmpv6-bytes { type yang:zero-based-counter64; - description - "An IPv6 packet which was not a fragment was received - consequently, it - did not need to be reassembled. This should be the usual case."; + description "Internally generted ICMPv6 error packets."; } - - leaf drop-ipv6-frag-invalid-reassembly { + leaf out-icmpv6-packets { type yang:zero-based-counter64; - description - "Two or more IPv6 fragments were received, and reassembly was started, - but was invalid and dropped. Causes include multiple fragments claiming - they are the last fragment, overlapping fragment offsets, or the packet - was being reassembled from too many fragments (the setting is - max_fragments_per_reassembly_packet, and the default is that no packet - should be reassembled from more than 40)."; + description "Internally generted ICMPv6 error packets."; } - - leaf drop-ipv6-frag-random-evicted { + leaf out-ipv4-bytes { type yang:zero-based-counter64; - description - "Reassembling an IPv6 packet from fragments was in progress, but the - configured amount of packets to reassemble at once was exceeded, so one - was dropped at random. Consider increasing the setting - max_ipv6_reassembly_packets."; + description "Valid outgoing IPv4 packets."; + } + leaf out-ipv4-frag { + type yang:zero-based-counter64; + description + "An outgoing packet exceeded the configured IPv4 MTU, so needed to be + fragmented. This may happen, but should be unusual."; + } + leaf out-ipv4-frag-not { + type yang:zero-based-counter64; + description + "An outgoing packet was small enough to pass through unfragmented - this + should be the usual case."; + } + leaf out-ipv4-packets { + type yang:zero-based-counter64; + description "Valid outgoing IPv4 packets."; + } + leaf out-ipv6-bytes { + type yang:zero-based-counter64; + description "All valid outgoing IPv6 packets."; } - leaf out-ipv6-frag { type yang:zero-based-counter64; - description + description "An outgoing packet exceeded the configured IPv6 MTU, so needed to be fragmented. This may happen, but should be unusual."; } - leaf out-ipv6-frag-not { type yang:zero-based-counter64; - description + description "An outgoing packet was small enough to pass through unfragmented - this should be the usual case."; } - - leaf memuse-ipv6-frag-reassembly-buffer { + leaf out-ipv6-packets { type yang:zero-based-counter64; - description - "The amount of memory being used by the statically sized data structure - for reassembling IPv6 fragments. This is directly proportional to the - setting max_ipv6_reassembly_packets."; + description "All valid outgoing IPv6 packets."; } } } From 6373ca03e1016a380c40a52e482d15ee55933e12 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 15:47:34 +0100 Subject: [PATCH 366/631] "snabb config" queries leader for features Before, "snabb config" wouldn't be able to parse ietf-softwire data as it didn't know which features were available. --- src/apps/config/leader.lua | 4 +++- src/lib/yang/schema.lua | 23 +++++++++++++++++++---- src/lib/yang/snabb-config-leader-v1.yang | 5 +++++ src/program/config/common.lua | 5 +++-- src/program/lwaftr/setup.lua | 4 ++-- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index fe3aba1421..206c16d4d4 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -9,6 +9,7 @@ local cltable = require("lib.cltable") local yang = require("lib.yang.yang") local data = require("lib.yang.data") local util = require("lib.yang.util") +local schema = require("lib.yang.schema") local rpc = require("lib.yang.rpc") local state = require("lib.yang.state") local path_mod = require("lib.yang.path") @@ -109,7 +110,8 @@ function Leader:rpc_describe (args) table.insert(alternate_schemas, schema_name) end return { native_schema = self.schema_name, - alternate_schema = alternate_schemas } + alternate_schema = alternate_schemas, + capability = schema.get_default_capabilities() } end local function path_printer_for_grammar(grammar, path) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index 1050c7b467..a61f8d9eb9 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -591,10 +591,25 @@ local function inherit_config(schema, config) end local default_features = {} --- Features should be a table whose keys are module names and whose --- values are feature name -> boolean tables. -function set_default_features(features) - default_features = features +function get_default_capabilities() + local ret = {} + for mod,features in pairs(default_features) do + local feature_names = {} + for feature,_ in pairs(features) do + table.insert(feature_names, feature) + end + ret[mod] = { feature = feature_names } + end + return ret +end +function set_default_capabilities(capabilities) + default_features = {} + for mod,caps in pairs(capabilities) do + default_features[mod] = {} + for _,feature in ipairs(caps.feature) do + default_features[mod][feature] = true + end + end end -- Inline "grouping" into "uses". diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index 9b4d201e0e..0b6b4a3a31 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -16,6 +16,11 @@ module snabb-config-leader-v1 { output { leaf native-schema { type string; mandatory true; } leaf-list alternate-schema { type string; } + list capability { + key module; + leaf module { type string; } + leaf-list feature { type string; } + } } } diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 055d69e22a..88e02667d3 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -48,11 +48,12 @@ function parse_command_line(args, opts) ['revision-date']="r", revision="r"}) if #args == 0 then err() end ret.instance_id = table.remove(args, 1) + local descr = call_leader(ret.instance_id, 'describe', {}) if not ret.schema_name then if opts.require_schema then err("missing --schema arg") end - ret.schema_name = - call_leader(ret.instance_id, 'describe', {}).native_schema + ret.schema_name = descr.native_schema end + require('lib.yang.schema').set_default_capabilities(descr.capability) if opts.with_config_file then if #args == 0 then err("missing config file argument") end local file = table.remove(args, 1) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index f6237087bb..098676225f 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -18,8 +18,8 @@ local ipv4 = require("lib.protocol.ipv4") local ethernet = require("lib.protocol.ethernet") local ipv4_ntop = require("lib.yang.util").ipv4_ntop -local lwaftr_yang_features = {['ietf-softwire']={binding=true, br=true}} -require('lib.yang.schema').set_default_features(lwaftr_yang_features) +local capabilities = {['ietf-softwire']={feature={'binding', 'br'}}} +require('lib.yang.schema').set_default_capabilities(capabilities) local function convert_ipv4(addr) if addr ~= nil then return ipv4:pton(ipv4_ntop(addr)) end From ff6e330efe0dbd72a1692c69457482b759b14188 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 16:37:47 +0100 Subject: [PATCH 367/631] Fix IPv4 and IPv6 prefix types --- src/lib/yang/value.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/value.lua b/src/lib/yang/value.lua index 3027907a74..4b624bf47b 100644 --- a/src/lib/yang/value.lua +++ b/src/lib/yang/value.lua @@ -126,7 +126,7 @@ types['mac-address'] = { } types['ipv4-prefix'] = { - ctype = 'struct { uint8_t[4] prefix; uint8_t len; }', + ctype = 'struct { uint8_t prefix[4]; uint8_t len; }', parse = function(str, what) local prefix, len = str:match('^([^/]+)/(.*)$') return { ipv4_pton(prefix), util.tointeger(len, 1, 32) } @@ -135,7 +135,7 @@ types['ipv4-prefix'] = { } types['ipv6-prefix'] = { - ctype = 'struct { uint8_t[16] prefix; uint8_t len; }', + ctype = 'struct { uint8_t prefix[16]; uint8_t len; }', parse = function(str, what) local prefix, len = str:match('^([^/]+)/(.*)$') return { assert(ipv6:pton(prefix)), util.tointeger(len, 1, 128) } From 4c04f3499df2b75ec75d3c148f580d9f2d20b95a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 16:38:05 +0100 Subject: [PATCH 368/631] Features are only available if they are in capabilities --- src/lib/yang/schema.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index a61f8d9eb9..a28bbe40ff 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -732,6 +732,9 @@ function resolve(schema, features) end if node.kind == 'feature' then node.module_id = lookup(env, 'module_id', '_') + if not (features[node.module_id] or {})[node.id] then + node.unavailable = true + end end for _,feature in ipairs(pop_prop(node, 'if_features') or {}) do local feature_node = lookup_lazy(env, 'features', feature) From c1e7eec40dde6ba21c0c8e8b594aaea60d3e05c3 Mon Sep 17 00:00:00 2001 From: Christian Graf Date: Mon, 5 Dec 2016 16:55:04 +0100 Subject: [PATCH 369/631] incorporating Fix formatting in issue 610 #8 to README.troubleshooting.md --- .../snabbvmx/doc/README.troubleshooting.md | 292 ++++++++++-------- 1 file changed, 169 insertions(+), 123 deletions(-) diff --git a/src/program/snabbvmx/doc/README.troubleshooting.md b/src/program/snabbvmx/doc/README.troubleshooting.md index 91322072d3..59289f13b1 100644 --- a/src/program/snabbvmx/doc/README.troubleshooting.md +++ b/src/program/snabbvmx/doc/README.troubleshooting.md @@ -278,10 +278,10 @@ The test goes through several steps: 5. Outgoing packets from the VM are mirrored to a tap interface (tap0). 6. Capture **responses on tap0** and compare them with the expected results. -The input data, as well as the expected output, is at `program/snabbvmx/tests/pcap`. +The input data, as well as the expected outputs, is at `program/snabbvmx/tests/pcap`. The test validates VLAN packets too. However, there are no VLAN tagged versions -of the expected output. The reason is that it is the NIC which tags and untags +of the expected outputs. The reason is that it is the NIC which tags and untags a packet. Since the packet did not leave the NIC yet, they come out from the VM untagged. @@ -324,33 +324,49 @@ NOTE: Currently the test is not working correctly: the returned MAC should be ### Brief steps -(1) run the interactive check with "-r" param to derive the counters and out.pcaps +1) Run the interactive check with "-r" param to derive the counters and out.pcaps: -`snabb snabbvmx check -r ./CONF.cfg "./V4-IN.PCAP" "./V6-IN.PCAP" "./outv4.pcap" "./outv6.pcap" COUNTERS.lua` - -(2) place derived counters.lua in "snabb/src/program/snabbvmx/tests/end-to-end/data/counters" +``` +$ snabb snabbvmx check -r ./CONF.cfg "./V4-IN.PCAP" "./V6-IN.PCAP" \ + "./outv4.pcap" "./outv6.pcap" COUNTERS.lua` +``` -(3) place derived and expected out.pcaps in "snabb/src/program/snabbvmx/tests/end-to-end/data" +2) Place derived counters.lua in "snabb/src/program/snabbvmx/tests/end-to-end/data/counters". -(4) edit the "test_env.sh" in snabb/src/program/snabbvmx/tests/end-to-end to have the test scripted +3) Place derived and expected out.pcaps in "snabb/src/program/snabbvmx/tests/end-to-end/data". -(5) run the scripted test: +4) Edit the "test_env.sh" in snabb/src/program/snabbvmx/tests/end-to-end +to have the test scripted. -`snabb/src/program/snabbvmx/tests/end-to-end$ sudo ./end-to-end.sh` +5) Run the scripted test: +``` +$ snabb/src/program/snabbvmx/tests/end-to-end$ sudo ./end-to-end.sh +``` ### How the system test works in detail -The way the test system works is by passing the input IPv4/IPv6 packets (via pre-recorded pcap-files) to the lwAFTR and comparing the expected packet output (pcap-file) to the packet output that the lwAFTR has generated. -The optional counters file is compared too to the actual counters file obtained after running the test. This is beeing reflected in snabbvmx check syntax: +The way the test system works is by passing the input IPv4/IPv6 packets (via +pre-recorded pcap-files) to the lwAFTR and comparing the expected packet output +(pcap-file) to the packet output that the lwAFTR has generated. -`CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP [COUNTERS.LUA]` +The optional counters file is compared too to the actual counters file obtained +after running the test. This is being reflected in snabbvmx check syntax: -V4-OUT.PCAP, V6-OUT.PCAP and COUNTERS.LUA are expected output. These output is compared to files stored temporarily in /tmp/endoutv4.pcap, /tmp/endoutv6.pcap and /tmp/counters.lua. These temporal files are the actual output produced by the lwAFTR after running a test. In order for a test to pass, the actual output must match the expected output. So it's not only that the counters file should match, but also the output .pcap files. (read: if a counters.lua file is provided, then it must still match the V4-OUT.PCAP and V6-OUT.PCAP) +`CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP [COUNTERS.LUA]` +V4-OUT.PCAP, V6-OUT.PCAP and COUNTERS.LUA are expected outputs. These outputs are +compared to files stored temporarily in /tmp/endoutv4.pcap, /tmp/endoutv6.pcap +and /tmp/counters.lua. These temporary files are the actual output produced by +the lwAFTR after running a test. In order for a test to pass, the actual output +must match the expected outputs. So it's not only that the counters file should +match, but also the output .pcap files. (read: if a counters.lua file is +provided, then it must still match the V4-OUT.PCAP and V6-OUT.PCAP). **lwaftr vs snabbvmx** -As both lwaftr and snabbvmx provide a different functionality and use different config-files, both the lwaftr and snabbvmx have their dedicated end-to-end tests. + +As both lwaftr and snabbvmx provide a different functionality and use different +config-files, both the lwaftr and snabbvmx have their dedicated end-to-end tests. Although SnabbVMX works on a single interface, `snabbvmx check` requires that the packet split (IPv4 / IPv6) is already done and provides a split output too. @@ -383,15 +399,21 @@ cd src/program/snabbvmx/tests/end-to-end ``` **interactive and scripted tests** -To develop an end-to-end tests, it's recommended to first run it interactively. Once the config-files, pcaps and counters are derived, the test can be added to the scripted tests in test_env.sh. + +To develop an end-to-end tests, it's recommended to first run it interactively. +Once the config-files, pcaps and counters are derived, the test can be added to +the scripted tests in test_env.sh. **interactive** -To run an interactive end-to-end test, either use the snabbvmx or lwaftr app - Have in mind that the test is running with the app specified (lwaftr or snabbvmx) -- snabb snabbvmx check -- snabb lwaftr check +To run an interactive end-to-end test, either use the snabbvmx or lwaftr app. Keep +in mind that the test is running with the app specified (lwaftr or snabbvmx). + +- snabb snabbvmx check. +- snabb lwaftr check. **end-to-end interactive usage** + ``` $ sudo ./snabb snabbvmx check Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP @@ -400,13 +422,14 @@ Usage: check [-r] CONF V4-IN.PCAP V6-IN.PCAP V4-OUT.PCAP V6-OUT.PCAP Parameters: -- **CONF** : SnabbVMX (icmp_snabbvmx-lwaftr-xe.cfg) or lwaftr (icmp_snabbvmx-lwaftr-xe1.conf) configuration file. -- **V4-IN.PCAP** : Incoming IPv4 packets (from Internet). -- **V6-IN.PCAP** : Incoming IPv6 packets (from b4). -- **V4-OUT.PCAP** : Outgoing IPv4 packets (to Internet, decapsulated). -- **V6-OUT.PCAP** : Outgoing IPv6 packets (to b4, encapsulated) -- **[COUNTERS.LUA]** : Lua file with counter values. Will be regenerated via [-r] param - +- **CONF**: SnabbVMX (icmp_snabbvmx-lwaftr-xe.cfg) or lwaftr +(icmp_snabbvmx-lwaftr-xe1.conf) configuration file. +- **V4-IN.PCAP**: Incoming IPv4 packets (from Internet). +- **V6-IN.PCAP**: Incoming IPv6 packets (from b4). +- **V4-OUT.PCAP**: Outgoing IPv4 packets (to Internet, decapsulated). +- **V6-OUT.PCAP**: Outgoing IPv6 packets (to b4, encapsulated) +- **[COUNTERS.LUA]**: Lua file with counter values. Will be regenerated via +[-r] param. ## How to run SnabbVMX interactive end-to-end test @@ -416,24 +439,30 @@ configuration and binding table. With that information and knowing the error report (ping to lwAFTR but it doesn't reply, valid softwire packet doesn't get decapsulated, etc), you craft a hand-made packet that meets the testing case. -**obtaining the config-files** +**Obtaining the config-files** To run a test, the following config-files are required: -- the binding-table : binding_table.txt.s -- lwaftr conf : snabbvmx-lwaftr-xe[0-9].conf -- snabbvmx cfg : snabbvmx-lwaftr-xe[0-9].cfg +- binding-table: binding_table.txt.s. +- lwaftr conf: snabbvmx-lwaftr-xe[0-9].conf. +- snabbvmx cfg: snabbvmx-lwaftr-xe[0-9].cfg. -If you are running lwaftr check, then snabbvmx config-file (snabbvmx-lwaftr-xe[0-9].cfg) is not required +If you are running lwaftr check, then snabbvmx config-file +(snabbvmx-lwaftr-xe[0-9].cfg) is not required. It is fine to copy or manually craft the config-files. -A running snabbvmx can be used as well to copy the config-files from the running container -To gain the used config-files from the running container, either run the collect-support-infos.sh (https://github.com/mwiget/vmxlwaftr/blob/igalia/SUPPORT-INFO.md) -or execute a shell within the dockers container and copy configs and binding-table from the /tmp directory. -Note: the check application is just using a single interface. If the running container consists of two or more snabb-instances, then just take one of them for when running the check. +A running snabbvmx can be used as well to copy the config-files from the running +container. To obtain the used config-files from the running container, either +run the collect-support-infos.sh (https://github.com/mwiget/vmxlwaftr/blob/ +igalia/SUPPORT-INFO.md) or execute a shell within the dockers container and copy +configs and binding-table from the /tmp directory. + +Note: The `snabbvmx check` application is just using a single interface. If the +running container consists of two or more snabb-instances, then just take one +of them for when running the check. -**collect-support-infos.sh** +**Script collect-support-infos.sh** ``` lab@ubuntu1:~/vmxlwaftr/tests$ ./collect-support-infos.sh lwaftr3-16.2R3 @@ -474,28 +503,44 @@ lab@ubuntu1:~/vmxlwaftr/tests/t1$ tar -tvzf support-info-20161108-1335.tgz -rwxr-xr-x root/root 2707019 2016-10-31 14:06 usr/local/bin/snabb ``` -**config-files within /tmp inside docker container** +**Config-files within /tmp inside docker container** -The snabbvmx config-files can be derived from the container's shell as well directly within the /tmp directory +The snabbvmx config-files can be derived from the container's shell as well +directly within the /tmp directory. ``` lab@ubuntu1:~/vmxlwaftr/tests/t1$ docker exec -ti lwaftr3-16.2R3 bash pid 2654's current affinity mask: fffff pid 2654's new affinity mask: ff3ff root@8f5d057b8298:/# ls /tmp/ -binding_table.txt config.new mac_xe0 snabb_xe0.log snabbvmx-lwaftr-xe1.cfg test-snabbvmx-lwaftr-xe0.cfg test_snabb_snabbvmx_xe0.sh vhost_features_xe1.socket -binding_table.txt.s config.new1 mac_xe1 snabb_xe1.log snabbvmx-lwaftr-xe1.conf test-snabbvmx-lwaftr-xe1.cfg test_snabb_snabbvmx_xe1.sh vmxhdd.img -binding_table.txt.s.new config.old pci_xe0 snabbvmx-lwaftr-xe0.cfg support-info.tgz test_snabb_lwaftr_xe0.sh vFPC-20160922.img xe0.socket -binding_table.txt.s.o junos-vmx-x86-64-16.1-20160926.0.qcow2 pci_xe1 snabbvmx-lwaftr-xe0.conf sysinfo.txt test_snabb_lwaftr_xe1.sh vhost_features_xe0.socket xe1.socket +binding_table.txt config.new +binding_table.txt.s config.new1 +binding_table.txt.s.new config.old +binding_table.txt.s.o junos-vmx.qcow2 +mac_xe snabb_xe0.log +mac_xe snabb_xe1.log +pci_xe snabbvmx-lwaftr-xe0.cfg +pci_xe snabbvmx-lwaftr-xe0.conf +snabbvmx-lwaftr-xe1.cfg test-snabbvmx-lwaftr-xe0.cfg +snabbvmx-lwaftr-xe1.conf test-snabbvmx-lwaftr-xe1.cfg +support-info.tgz test_snabb_lwaftr_xe0.sh +sysinfo.txt test_snabb_lwaftr_xe1.sh +test_snabb_snabbvmx_xe0.sh vhost_features_xe1.socket +test_snabb_snabbvmx_xe1.sh vmxhdd.img +vFPC-20160922.img xe0.socket +vhost_features_xe0.socket xe1.socket ``` -Note: press ctrl p ctrl q to exit the containers shell +Note: Press Ctrl-P + Ctrl-Q to exit the containers shell. -**some adoption of config-files is required** +**Some adoption of config-files is required** -The advantage of snabbvmx is the dynamic next-hop resolution via Junos. When running the the lwaftr or snabbvmx app standalone, then the next-hop resolution via Junos is missing. The config-files are required to get modified for static next-hop configuration. +The advantage of snabbvmx is the dynamic next-hop resolution via Junos. When +running the lwaftr or snabbvmx app standalone, then the next-hop resolution +via Junos is missing. The config-files are required to get modified for static +next-hop configuration. -**config as derived from a running vmxlwaftr container** +**Config as derived from a running vmxlwaftr container** ``` lab@ubuntu1:~/vmxlwaftr/tests/t1$ cat snabbvmx-lwaftr-xe1.cfg @@ -520,12 +565,14 @@ return { To change configuration for static next-hop, below changes are required: +- cache_refresh_interval = 0 (turns off next-hop learning via Junos). +- mac_address (thats the own/self mac-address fo lwaftr). +- next_hop_mac (next-hop mac to send the packets to). -- cache_refresh_interval = 0 (turns off next-hop learning via Junos) -- mac_address (thats the own/self mac-address fo lwaftr) -- next_hop_mac (next-hop mac to send the packets to) - -Note: The snabbvmx config icmp_snabbvmx-lwaftr-xe1.cfg references the snabb-configuration file icmp_snabbvmx-lwaftr-xe1.conf via the "lwaftr" directive. When running "snabbvmx check" then both the lwaftr and the snabbvmx config-files must be provided. +Note: The snabbvmx config icmp_snabbvmx-lwaftr-xe1.cfg references the +snabb-configuration file icmp_snabbvmx-lwaftr-xe1.conf via the "lwaftr" directive. +When running "snabbvmx check" then both the lwaftr and the snabbvmx config-files +must be provided. ``` cd snabb/src/program/snabbvmx/tests/end-to-end/data$ @@ -536,46 +583,47 @@ return { }, ipv6_interface = { ipv6_address = "", - cache_refresh_interval = 0, <<< set this to 0 + cache_refresh_interval = 0, # Set this to 0 fragmentation = false, - mac_address = "02:cf:69:15:81:01", <<< the lwaftr's own mac address. input pcap match this mac - next_hop_mac = "90:e2:ba:94:2a:bc", <<< the next-hop mac to use for outgoing packets + mac_address = "02:cf:69:15:81:01", # lwaftr's own mac address. input pcap match this mac + next_hop_mac = "90:e2:ba:94:2a:bc", # the next-hop mac to use for outgoing packets }, ipv4_interface = { ipv4_address = "192.168.5.2", cache_refresh_interval = 0, <<< set this to 0 fragmentation = false, - mac_address = "02:cf:69:15:81:01", <<< the lwaftr's own mac address. input pcap match this mac - next_hop_mac = "90:e2:ba:94:2a:bc", <<< the next-hop mac to use for outgoing packets + mac_address = "02:cf:69:15:81:01", # lwaftr's own mac address. input pcap match this mac + next_hop_mac = "90:e2:ba:94:2a:bc", # the next-hop mac to use for outgoing packets }, } ``` **The input pcaps** -The snabb "check" app requires one or two input pcaps. -It is ok to: +The snabb "check" app requires one or two input pcaps. It is OK to: -- only feed V4-IN.PCAP -- only feed the V6-IN.PCAP -- or feed both V4-IN.PCAP and V6-IN.PCAP +- Only feed V4-IN.PCAP. +- Only feed the V6-IN.PCAP. +- Feed both V4-IN.PCAP and V6-IN.PCAP. -Note for interactive tests:
-When only feeding one pcap as input, then the other empty pcap must be the "empty.pcap" in -"src/program/snabbvmx/tests/end-to-end/data/empty.pcap" and not an empty string like "" +Note for interactive tests: When only feeding one pcap as input, then the other +empty pcap must be the "empty.pcap" (src/program/snabbvmx/tests/end-to-end/ +data/empty.pcap) and not an empty string like "". ``` cd snabb/src/program/snabbvmx/tests/end-to-end/data$ $ file empty.pcap -empty.pcap: tcpdump capture file (little-endian) - version 2.4 (Ethernet, capture length 65535) +empty.pcap: tcpdump capture file (little-endian) - version 2.4 (Ethernet, +capture length 65535) $ tcpdump -r empty.pcap reading from file empty.pcap, link-type EN10MB (Ethernet) ``` -**sample icmp-ipv4-in.pcap** +**Sample icmp-ipv4-in.pcap** -For any input pcap it makes sense to keep it short - ideally a single packet to check correctness of the lwaftr. -The input packet "11:26:07.168372 IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain]" is matching the binding-table for PSID=1. +For any input pcap it makes sense to keep it short - ideally a single packet to +check correctness of the lwaftr. The input packet "11:26:07.168372 IP +10.0.1.100.53 > 10.10.0.0.1024: [|domain]" is matching the binding-table for PSID=1. ``` cd snabb/src/program/snabbvmx/tests/end-to-end/data$ @@ -600,13 +648,16 @@ reading from file ipv4-in.pcap, link-type EN10MB (Ethernet) 11:26:07.168372 IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] ``` -**running the interactive check** +**Running the interactive check** -The interactive check is performed via `snabb snabbvmx check -r`. Using the "-r" param instructs the check to generate the counters and out.pcap files. +The interactive check is performed via `snabb snabbvmx check -r`. Using the +"-r" param instructs the check to generate the counters and out.pcap files. As there is no IPv6 input, the "empty.pcap" is configured. ``` -sudo ~/vmx-docker-lwaftr/snabb/src/snabb snabbvmx check -r ./snabbvmx-lwaftr-xe1.cfg "./ipv4-in.pcap" "./empty.pcap" "./outv4.pcap" "./outv6.pcap" test.lua +sudo ~/vmx-docker-lwaftr/snabb/src/snabb snabbvmx check -r \ + ./snabbvmx-lwaftr-xe1.cfg "./ipv4-in.pcap" "./empty.pcap" \ + "./outv4.pcap" "./outv6.pcap" test.lua loading compiled binding table from ./binding_table.txt.s.o compiled binding table ./binding_table.txt.s.o is up to date. nh_fwd4: cache_refresh_interval set to 0 seconds @@ -616,15 +667,17 @@ nh_fwd6: static next_hop_mac 90:e2:ba:94:2a:bc done ``` -**results** +**Results** -The -r flag is set, as such the resulting counters file test.lua and the out.pcap files are generated freshly. -The results below show a correct processing of the lwatr: +The -r flag is set, as such the resulting counters file test.lua and the +out.pcap files are generated freshly. The results below show a correct +processing of the lwatr: -- the counters-file lists one IPv4 packet as input -- as the input-packet matches the binding-table, one IPv6 packet output -- outv4.pcap is empty -- outv6.pcap shows the resulting encapsulated lw4o6 packet +- The counters-file lists one IPv4 packet as input as the input-packet matches +the binding-table. +- One IPv6 packet output. +- Outv4.pcap is empty. +- Outv6.pcap shows the resulting encapsulated lw4o6 packet. ``` $ cd snabb/src/program/snabbvmx/tests/end-to-end/data @@ -640,46 +693,46 @@ reading from file outv4.pcap, link-type EN10MB (Ethernet) $ tcpdump -n -r outv6.pcap reading from file outv6.pcap, link-type EN10MB (Ethernet) -01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::400: IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] - +01:00:00.000000 IP6 2a02:587:f700::100 > 2a02:587:f710::400: +IP 10.0.1.100.53 > 10.10.0.0.1024: [|domain] ``` -**summary interactive check** - -At this stage the interactive test is finished. -The following is defined: +**Summary of interactive check** -- lwaftr and snabbvmx configs -- binding-table -- input pcaps -- expected resulting counters and out.pcap +At this stage the interactive test is finished. The following is defined: -With known input and results, the test can now be added to the scripted end-to-end.sh - to be executed with all other tests to ensure snabbvmx behaves as it should. +- lwaftr and snabbvmx configs. +- Binding-table. +- Input pcaps. +- Expected resulting counters and out.pcap. -Further more, this end-to-end procedure can be ideally used to report issues! - -Tip: packets always arrive only in one interface, but the output might be -empty or non-empty for both IPv4 and IPv6. +With known input and results, the test can now be added to the end-to-end.sh +script to be executed with all other tests to ensure snabbvmx behaves as it +should. Furthermore, this end-to-end procedure can be ideally used to report +issues! +Tip: Packets always arrive only in one interface, but the output might be empty +or non-empty for both IPv4 and IPv6. ## Adding the sample-test towards the scripted end-to-end tests ### In short -- place counter file and out.pcaps into correct directory -- edit "test_env.sh" and add the test -- run the test +- Place counter file and out.pcaps into correct directory. +- Edit "test_env.sh" and add the test. +- Run the test. ### Detailed steps -**directory-structure** -place out.pcap into the data-drectory +Step 1. Directory structure: + +Place out.pcap into the data-drectory ``` snabb/src/program/snabbvmx/tests/end-to-end/data ``` -place counter-file into counters-drectory +Place counter-file into counters-drectory ``` snabb/src/program/snabbvmx/tests/end-to-end/data/counters @@ -695,17 +748,16 @@ configuration files and binding tables. * **end-to-end-vlan.sh**: Runs **core-end-to-end.sh** on VLAN packets. * **selftest.sh**: Runs both **end-to-end.sh** and **end-to-end-vlan.sh** +Step 2. Adding the sample test: -**adding the sample test**: +All tests are defined in the "test_env.sh" file. This "test_env.sh" file has +to be edited to include the sample test. The check app always checks the +provided data to decide if the test is successful or not: -All tests are defined in the "test_env.sh" file. -This "test_env.sh" file has to be edited to include the sample test. -The check app always checks the provided data to decide if the test is successful or not:
+- If the optional counters-file is provided, then it must match. +- The resulting out.pcap files must always match. -- if the optional counters-file is provided, then it must match -- the resulting out.pcap files must always match - -**pass-criteria without the optional counters.lua file** +Step 3. Pass-criteria without the optional counters.lua file: ``` src/program/snabbvmx/tests/end-to-end$ vi test_env.sh @@ -727,8 +779,9 @@ TEST_DATA=( "snabbvmx-lwaftr-xe1.cfg" "" "regressiontest-signedntohl-frags.pcap" "" "" "drop-all-ipv6-fragments.lua" ) +``` - +``` src/program/snabbvmx/tests/end-to-end$ sudo ./end-to-end.sh Testing: sample test loading compiled binding table from data/binding_table.txt.s.o @@ -751,24 +804,17 @@ Test passed All end-to-end lwAFTR tests passed. ``` -If the counters file shall be taken into consideration as well (e.g. to count dropped frames), then just one line needs to be changed from "" to the counters file "test.lua" +If the counters file shall be taken into consideration as well (e.g. to count +dropped frames), then just one line needs to be changed from "" to the counters +file "test.lua". ``` $ cd snabb/src/program/snabbvmx/tests/end-to-end $ vi test_env.sh -... - "sample test" - "snabbvmx-lwaftr-xe1.cfg" "ipv4-in.pcap" "" "" "outv6.pcap" - "test.lua" ``` - - - - - - - - - - +``` +"sample test" +"snabbvmx-lwaftr-xe1.cfg" "ipv4-in.pcap" "" "" "outv6.pcap" +"test.lua" +``` From 4f4f34e457e8cb09cd3cb0fedd795209c86bab50 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 5 Dec 2016 17:03:09 +0100 Subject: [PATCH 370/631] Do not show an empty link report in run --- src/program/lwaftr/selftest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh index b3f292a7fa..7133693df3 100755 --- a/src/program/lwaftr/selftest.sh +++ b/src/program/lwaftr/selftest.sh @@ -22,6 +22,7 @@ echo "Testing snabb lwaftr bench --reconfigurable" LEADER_PID=$! +sleep 0.1 ./snabb config get $LEADER_PID / From 6ba9d8c67ccfbde69c344c458b9aceeacd482cc7 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 17:07:40 +0100 Subject: [PATCH 371/631] Fix data parsing bugs Keys to cltables should be unique, and fix bug for keys for whom normalize_id(id) ~= id. --- src/lib/yang/data.lua | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index ce15047b9e..17baaad85a 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -282,7 +282,10 @@ end local function cltable_builder(key_t) local res = cltable.new({ key_type=key_t }) local builder = {} - function builder:add(key, value) res[key] = value end + function builder:add(key, value) + assert(res[key] == nil, 'duplicate key') + res[key] = value + end function builder:finish() return res end return builder end @@ -322,9 +325,13 @@ local function table_parser(keyword, keys, values, string_key, key_ctype, local key, value = {}, {} if key_t then key = key_t() end if value_t then value = value_t() end - for k,v in pairs(struct) do + for k,_ in pairs(keys) do + local id = normalize_id(k) + key[id] = struct[id] + end + for k,_ in pairs(values) do local id = normalize_id(k) - if keys[k] then key[id] = v else value[id] = v end + value[id] = struct[id] end assoc:add(key, value) return assoc From 74d5758b09d2511a67fce565c1893c02f7b44617 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 17:08:15 +0100 Subject: [PATCH 372/631] Fix "snabb config load" --- src/program/config/common.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 88e02667d3..30037cec2f 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -96,7 +96,7 @@ function open_socket_or_die(instance_id) end function serialize_config(config, schema_name, path) - local grammar = path_grammar(schema_name, path) + local grammar = path_grammar(schema_name, path or '/') local printer = data.data_printer_from_grammar(grammar) return printer(config, yang.string_output_file()) end From a7072dce20495c268e2afd1c39dc6078652177a5 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 17:08:43 +0100 Subject: [PATCH 373/631] First pass at updates in terms of ietf-softwire Not yet working. --- src/apps/config/leader.lua | 15 +- src/apps/config/support/snabb-softwire-v1.lua | 327 +++++++++++++++++- 2 files changed, 327 insertions(+), 15 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 206c16d4d4..3bd0fc60d0 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -388,7 +388,14 @@ function compute_remove_config_fn (schema_name, path) return path_remover_for_schema(yang.load_schema_by_name(schema_name), path) end -function Leader:update_configuration (schema_name, update_fn, verb, path, ...) +function Leader:notify_pre_update (config, verb, path, ...) + for _,translator in pairs(self.support.translators) do + translator.pre_update(config, verb, path, ...) + end +end + +function Leader:update_configuration (update_fn, verb, path, ...) + self:notify_pre_update(self.current_configuration, verb, path, ...) local to_restart = self.support.compute_apps_to_restart_after_configuration_update ( self.schema_name, self.current_configuration, verb, path, @@ -409,8 +416,7 @@ end function Leader:handle_rpc_update_config (args, verb, compute_update_fn) local path = path_mod.normalize_path(args.path) local parser = path_parser_for_schema_by_name(args.schema, path) - self:update_configuration(args.schema, - compute_update_fn(args.schema, path), + self:update_configuration(compute_update_fn(args.schema, path), verb, path, parser(args.config)) return {} end @@ -496,8 +502,7 @@ function Leader:rpc_remove_config (args) return self:foreign_rpc_remove_config(args.schema, args.path) end local path = path_mod.normalize_path(args.path) - self:update_configuration(args.schema, - compute_remove_config_fn(args.schema, path), + self:update_configuration(compute_remove_config_fn(args.schema, path), 'remove', path) return {} end diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index c9f711b515..ab318d759b 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -3,7 +3,10 @@ module(..., package.seeall) local ffi = require('ffi') local app = require('core.app') local data = require('lib.yang.data') +local ipv4_ntop = require('lib.yang.util').ipv4_ntop +local ipv6 = require('lib.protocol.ipv6') local yang = require('lib.yang.yang') +local ctable = require('lib.ctable') local cltable = require('lib.cltable') local path_mod = require('lib.yang.path') local generic = require('apps.config.support').generic_schema_config_support @@ -134,8 +137,8 @@ local function ietf_binding_table_from_native(bt) local v = { binding_ipv4_addr = entry.key.ipv4, port_set = { - psid_offset = psid_map.psid_offset, - psid_len = psid_map.psid_len, + psid_offset = psid_map.reserved_ports_bit_count, + psid_len = psid_map.psid_length, psid = entry.key.psid }, br_ipv6_addr = bt.br_address[entry.value.br+1], @@ -146,22 +149,77 @@ local function ietf_binding_table_from_native(bt) return ret end +local function schema_getter(schema_name, path) + local schema = yang.load_schema_by_name(schema_name) + local grammar = data.data_grammar_from_schema(schema) + return path_mod.resolver(grammar, path) +end + +local function snabb_softwire_getter(path) + return schema_getter('snabb-softwire-v1', path) +end + +local function ietf_softwire_getter(path) + return schema_getter('ietf-softwire', path) +end + +local function native_binding_table_from_ietf(ietf) + local _, psid_map_grammar = + snabb_softwire_getter('/softwire-config/binding-table/psid-map') + local psid_map_key_t = data.typeof(psid_map_grammar.key_ctype) + local psid_map = cltable.new({key_type=psid_map_key_t}) + local br_address = {} + local br_address_by_ipv6 = {} + local _, softwire_grammar = + snabb_softwire_getter('/softwire-config/binding-table/softwire') + local softwire_key_t = data.typeof(softwire_grammar.key_ctype) + local softwire_value_t = data.typeof(softwire_grammar.value_ctype) + local softwire = ctable.new({key_type=softwire_key_t, + value_type=softwire_value_t}) + for k,v in cltable.pairs(ietf) do + local br_address_key = ipv6:ntop(k.binding_ipv6info) + local br = br_address_by_ipv6[br_address_key] + if not br then + br = #br_address + table.insert(br_address, k.binding_ipv6info) + br_address_by_ipv6[br_address_key] = br + end + local psid_key = psid_map_key_t({addr=v.binding_ipv4_addr}) + if not psid_map[psid_key] then + psid_map[psid_key] = {psid_length=v.port_set.psid_len, + reserved_ports_bit_count=v.port_set.psid_offset} + end + softwire:add(softwire_key_t({ipv4=v.binding_ipv4_addr, + psid=v.port_set.psid}), + softwire_value_t({br=br, b4_ipv6=v.binding_ipv6info})) + end + return {psid_map=psid_map, br_address=br_address, softwire=softwire} +end + +local function serialize_binding_table(bt) + local _, grammar = snabb_softwire_getter('/softwire-config/binding-table') + local printer = data.data_printer_from_grammar(grammar) + return printer(bt, yang.string_output_file()) +end + local function ietf_softwire_translator () local ret = {} + local cached_config function ret.get_config(native_config) - -- Such nesting, very standard, wow + if cached_config ~= nil then return cached_config end local br_instance, br_instance_key_t = cltable_for_grammar(get_ietf_br_instance_grammar()) br_instance[br_instance_key_t({id=1})] = { -- FIXME tunnel_payload_mtu = 0, tunnel_path_mru = 0, + softwire_num_threshold = 0xffffffff, binding_table = { binding_entry = ietf_binding_table_from_native( native_config.softwire_config.binding_table) } } - return { + cached_config = { softwire_config = { binding = { br = { @@ -170,8 +228,8 @@ local function ietf_softwire_translator () } } } + return cached_config end - ret.get_config = memoize1(ret.get_config) function ret.get_state(native_state) -- Even though this is a different br-instance node, it is a -- cltable with the same key type, so just re-use the key here. @@ -196,15 +254,264 @@ local function ietf_softwire_translator () } } end - ret.get_state = memoize1(ret.get_state) - function ret.set_config(native_config, path, data) - error('unimplemented') + local function sets_whole_table(path, count) + if #path > count then return false end + if #path == count then + for k,v in pairs(path[#path].query) do return false end + end + return true + end + function ret.set_config(native_config, path_str, arg) + path = path_mod.parse_path(path_str) + local br_instance_paths = {'softwire-config', 'binding', 'br', + 'br-instances', 'br-instance'} + local bt_paths = {'binding-table', 'binding-entry'} + -- Two kinds of updates: setting the whole binding table, or + -- updating one entry. + if sets_whole_table(path, #br_instance_paths + #bt_paths) then + -- Setting the whole binding table. + if sets_whole_table(path, #br_instance_paths) then + for i=#path+1,#br_instance_paths do + arg = arg[data.normalize_id(br_instance_paths[i])] + end + local instance + for k,v in cltable.pairs(arg) do + if instance then error('multiple instances in config') end + if k.id ~= 1 then error('instance id not 1: '..tostring(k.id)) end + instance = v + end + if not instance then error('no instances in config') end + arg = instance + end + for i=math.max(#path-#br_instance_paths,0)+1,#bt_paths do + arg = arg[data.normalize_id(bt_paths[i])] + end + local bt = native_binding_table_from_ietf(arg) + return {{'set', {schema='snabb-softwire-v1', + path='/softwire-config/binding-table', + config=serialize_binding_table(bt)}}} + else + -- An update to an existing entry. First, get the existing entry. + local config = ret.get_config(native_config) + local entry_path = path_str + local entry_path_len = #br_instance_paths + #br_paths + for i=entry_path_len+1, #path do + entry_path = lib.dirname(entry_path) + end + local old = ietf_softwire_getter(entry_path)(config) + -- Now figure out what the new entry should look like. + local new + if #path == entry_path_len then + new = arg + else + new = { + port_set = { + psid_offset = entry.port_set.psid_offset, + psid_len = entry.port_set.psid_len, + psid = entry.port_set.psid + }, + binding_ipv4_addr = entry.binding_ipv4_addr, + br_ipv6_addr = entry.br_ipv6_addr + } + if path[#entry_path_len + 1] == 'port-set' then + if #path == #entry_path_len + 1 then + new.port_set = arg + else + local k = data.normalize_id(path[#path]) + new.port_set[k] = arg + end + elseif path[#path] == 'binding-ipv4-addr' then + new.binding_ipv4_addr = arg + elseif path[#path] == 'br-ipv6-addr' then + new.br_ipv6_addr = arg + else + error('bad path element: '..path[#path]) + end + end + -- Apply changes. Start by ensuring that there's a br-address + -- entry for this softwire, and ensuring that the port-set + -- changes are compatible with the existing configuration. + local updates = {} + local softwire_path = '/softwire-config/binding-table/softwire' + local psid_map_path = '/softwire-config/binding-table/psid-map' + local br_address_path = '/softwire-config/binding-table/br-address' + local new_br + local bt = native_config.softwire_config.binding_table + for i,br in ipairs(bt.br_address) do + if lib.equal(br, new.br_ipv6_addr) then + new_br = i - 1; break + end + end + if new_br == nil then + new_br = #bt.br_address + table.insert(updates, + {'add', {schema='snabb-softwire-v1', + path=br_address_path, + config=ipv6:ntop(new.br_ipv6_addr)}}) + end + if new.binding_ipv4_addr ~= old.binding_ipv4_addr then + local psid_key_t = data.typeof('struct { uint32_t ipv4; }') + local psid_map_entry = + bt.psid_map[psid_key_t({ipv4=new.binding_ipv4_addr})] + if psid_map_entry == nil then + local config_str = string.format( + "{ addr %s; psid-length %s; reserved-ports-bit-count %s; }", + ipv4_ntop(new.binding_ipv4_addr), new.port_set.psid_len, + new.port_set.psid_offset) + table.insert(updates, + {'add', {schema='snabb-softwire-v1', + path=psid_map_path, + config=config_str}}) + elseif (psid_map_entry.psid_length ~= new.port_set.psid_len or + psid_map_entry.reserved_ports_bit_count ~= + new.port_set.psid_offset) then + -- The Snabb lwAFTR is restricted to having the same PSID + -- parameters for every softwire on an IPv4. Here we + -- have a request to have a softwire whose paramters + -- differ from those already installed for this IPv4 + -- address. We would need to verify that there is no + -- other softwire on this address. + error('changing psid params unimplemented') + end + elseif (new.port_set.psid_offset ~= old.port_set.psid_offset or + new.port_set.psid_len ~= old.port_set.psid_len) then + -- See comment above. + error('changing psid params unimplemented') + end + -- OK, psid_map and br_address taken care of, let's just remove + -- this softwire entry and add a new one. + local function q(ipv4, psid) + return string.format('[ipv4=%s][psid=%s]', ipv4_ntop(ipv4), psid) + end + local old_query = q(old.binding_ipv4_addr, old.port_set.psid) + table.insert(updates, + {'remove', {schema='snabb-softwire-v1', + path=softwire_path..old_query}}) + local config_str = string.format( + '{ ipv4 %s; psid %s; br %s; b4-ipv6 %s; }', + ipv4_ntop(new.binding_ipv4_addr), new.port_set.psid, + new_br, path[#entry_path_len].query['binding-ipv6info']) + table.insert(updates, + {'add', {schema='snabb-softwire-v1', + path=softwire_path, + config=config_str}}) + return updates + end end function ret.add_config(native_config, path, data) - error('unimplemented') + if path ~= ('/softwire-config/binding/br-instances'.. + '/br-instance[id=1]/binding-table/binding-entry') then + error('unsupported path: '..path) + end + local config = ret.get_config(native_config) + local ietf_bt = ietf_softwire_getter(path)(config) + local old_bt = native_config.softwire_config.binding_table + local new_bt = native_binding_table_from_ietf(ietf_bt) + -- Add new psid_map entries. + for k,v in cltable.pairs(new_bt.psid_map) do + if old_bt.psid_map[k] then + if not lib.equal(old_bt.psid_map[k], v) then + error('changing psid params unimplemented') + end + else + local config_str = string.format( + "{ addr %s; psid-length %s; reserved-ports-bit-count %s; }", + ipv4_ntop(k.addr), v.psid_length, v.reserved_ports_bit_count) + table.insert(updates, + {'add', {schema='snabb-softwire-v1', + path=psid_map_path, + config=config_str}}) + end + end + -- Remap br-address entries. + local br_address_map = {} + local br_address_count = #br_addresses + for _,new_br_address in ipairs(new_bt.br_addresses) do + local idx + for i,old_br_address in ipairs(old_bt.br_addresses) do + if lib.equal(old_br_address, new_br_address) then + idx = i - 1 -- zero-based indexes, fml + break + end + end + if not idx then + idx, br_address_count = br_address_count, br_address_count + 1 + table.insert(updates, + {'add', {schema='snabb-softwire-v1', + path=br_address_path, + config=ipv6:ntop(new_br_address)}}) + end + table.insert(br_address_map, idx) + end + -- Add softwires. + local additions = {} + for entry in new_bt.softwire:iterate() do + if old_bt.softwire.lookup_ptr(entry) then + error('softwire already present in table: '.. + inet_ntop(entry.key.ipv4)..'/'..entry.key.psid) + end + local config_str = string.format( + '{ ipv4 %s; psid %s; br %s; b4-ipv6 %s; }', + ipv4_ntop(entry.key.ipv4), entry.key.psid, + br_address_map[entry.value.br + 1], ipv6:ntop(entry.value.b4_ipv6)) + table.insert(additions, config_str) + end + table.insert(updates, + {'add', {schema='snabb-softwire-v1', + path=softwire_path, + config=table.concat(additions, '\n')}}) + return updates end function ret.remove_config(native_config, path) - error('unimplemented') + local ietf_binding_table_path = + '/softwire-config/binding/br-instances/br-instance[id=1]/binding-table' + if (lib.dirname(path) ~= ietf_binding_table_path or + path:sub(-1) ~= ']') then + error('unsupported path: '..path) + end + local config = ret.get_config(native_config) + local entry = ietf_softwire_getter(path)(config) + local function q(ipv4, psid) + return string.format('[ipv4=%s][psid=%s]', ipv4_ntop(ipv4), psid) + end + local query = q(entry.binding_ipv4_addr, entry.port_set.psid) + return {{'remove', {schema='snabb-softwire-v1', + path=softwire_path..query}}} + end + function ret.pre_update(native_config, path, verb, data) + -- Given the notification that the native config is about to be + -- updated, make our cached config follow along if possible (and + -- if we have one). Otherwise throw away our cached config; we'll + -- re-make it next time. + if cached_config == nil then return end + if (verb == 'remove' and + path:match('^/softwire%-config/binding%-table/softwire')) then + -- Remove a softwire. + local value = snabb_softwire_getter(path)(config) + local br = cached_config.softwire_config.binding.br + for _,instance in cltable.pairs(br.br_instances.br_instance) do + local grammar = get_softwire_grammar() + local key = path_mod.prepare_table_lookup( + grammar.keys, grammar.key_ctype, + {['binding-ipv6info']=value.b4_ipv6}) + assert(instance.binding_table.binding_entry[key] ~= nil) + instance.binding_table.binding_entry[key] = nil + end + elseif (verb == 'add' and + path == '/softwire-config/binding-table/softwire') then + local bt = native_config.softwire_config.binding_table + for k,v in cltable.pairs( + ietf_binding_table_from_native( + { psid_map = bt.psid_map, br_address = bt.br_address, + softwire = data })) do + local br = cached_config.softwire_config.binding.br + for _,instance in cltable.pairs(br.br_instances.br_instance) do + instance.binding_table.binding_entry[k] = v + end + end + else + cached_config = nil + end end return ret end From 12886da8a2bb923b1fb105fc4c78ce5abd8f63d4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 17:21:16 +0100 Subject: [PATCH 374/631] Fix creation of native binding table from IETF binding table --- src/apps/config/support/snabb-softwire-v1.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index ab318d759b..71661bea42 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -177,7 +177,7 @@ local function native_binding_table_from_ietf(ietf) local softwire = ctable.new({key_type=softwire_key_t, value_type=softwire_value_t}) for k,v in cltable.pairs(ietf) do - local br_address_key = ipv6:ntop(k.binding_ipv6info) + local br_address_key = ipv6:ntop(v.br_ipv6_addr) local br = br_address_by_ipv6[br_address_key] if not br then br = #br_address @@ -191,7 +191,7 @@ local function native_binding_table_from_ietf(ietf) end softwire:add(softwire_key_t({ipv4=v.binding_ipv4_addr, psid=v.port_set.psid}), - softwire_value_t({br=br, b4_ipv6=v.binding_ipv6info})) + softwire_value_t({br=br, b4_ipv6=k.binding_ipv6info})) end return {psid_map=psid_map, br_address=br_address, softwire=softwire} end From 15edc01adbe1f74d345a7d9fad72acd41487e5f3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Mon, 5 Dec 2016 18:37:57 +0100 Subject: [PATCH 375/631] Fix "add" on ietf-softwire --- src/apps/config/support/snabb-softwire-v1.lua | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 71661bea42..f58de87427 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -2,6 +2,7 @@ module(..., package.seeall) local ffi = require('ffi') local app = require('core.app') +local equal = require('core.lib').equal local data = require('lib.yang.data') local ipv4_ntop = require('lib.yang.util').ipv4_ntop local ipv6 = require('lib.protocol.ipv6') @@ -338,7 +339,7 @@ local function ietf_softwire_translator () local new_br local bt = native_config.softwire_config.binding_table for i,br in ipairs(bt.br_address) do - if lib.equal(br, new.br_ipv6_addr) then + if equal(br, new.br_ipv6_addr) then new_br = i - 1; break end end @@ -399,19 +400,23 @@ local function ietf_softwire_translator () end end function ret.add_config(native_config, path, data) - if path ~= ('/softwire-config/binding/br-instances'.. + if path ~= ('/softwire-config/binding/br/br-instances'.. '/br-instance[id=1]/binding-table/binding-entry') then error('unsupported path: '..path) end local config = ret.get_config(native_config) local ietf_bt = ietf_softwire_getter(path)(config) local old_bt = native_config.softwire_config.binding_table - local new_bt = native_binding_table_from_ietf(ietf_bt) + local new_bt = native_binding_table_from_ietf(data) + local updates = {} + local softwire_path = '/softwire-config/binding-table/softwire' + local psid_map_path = '/softwire-config/binding-table/psid-map' + local br_address_path = '/softwire-config/binding-table/br-address' -- Add new psid_map entries. for k,v in cltable.pairs(new_bt.psid_map) do if old_bt.psid_map[k] then - if not lib.equal(old_bt.psid_map[k], v) then - error('changing psid params unimplemented') + if not equal(old_bt.psid_map[k], v) then + error('changing psid params unimplemented: '..k.addr) end else local config_str = string.format( @@ -425,11 +430,11 @@ local function ietf_softwire_translator () end -- Remap br-address entries. local br_address_map = {} - local br_address_count = #br_addresses - for _,new_br_address in ipairs(new_bt.br_addresses) do + local br_address_count = #old_bt.br_address + for _,new_br_address in ipairs(new_bt.br_address) do local idx - for i,old_br_address in ipairs(old_bt.br_addresses) do - if lib.equal(old_br_address, new_br_address) then + for i,old_br_address in ipairs(old_bt.br_address) do + if equal(old_br_address, new_br_address) then idx = i - 1 -- zero-based indexes, fml break end @@ -446,7 +451,7 @@ local function ietf_softwire_translator () -- Add softwires. local additions = {} for entry in new_bt.softwire:iterate() do - if old_bt.softwire.lookup_ptr(entry) then + if old_bt.softwire:lookup_ptr(entry) then error('softwire already present in table: '.. inet_ntop(entry.key.ipv4)..'/'..entry.key.psid) end From 76fefdc9ce9a2368d19d3722bf225cc29e2d150c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 09:58:02 +0100 Subject: [PATCH 376/631] Bug fixes to ietf-softwire support Also a bug fix in an error message for table lookups --- src/apps/config/support/snabb-softwire-v1.lua | 11 ++++++----- src/lib/yang/path.lua | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index f58de87427..0c255569cc 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -3,6 +3,7 @@ module(..., package.seeall) local ffi = require('ffi') local app = require('core.app') local equal = require('core.lib').equal +local dirname = require('core.lib').dirname local data = require('lib.yang.data') local ipv4_ntop = require('lib.yang.util').ipv4_ntop local ipv6 = require('lib.protocol.ipv6') @@ -182,7 +183,7 @@ local function native_binding_table_from_ietf(ietf) local br = br_address_by_ipv6[br_address_key] if not br then br = #br_address - table.insert(br_address, k.binding_ipv6info) + table.insert(br_address, v.br_ipv6_addr) br_address_by_ipv6[br_address_key] = br end local psid_key = psid_map_key_t({addr=v.binding_ipv4_addr}) @@ -297,7 +298,7 @@ local function ietf_softwire_translator () local entry_path = path_str local entry_path_len = #br_instance_paths + #br_paths for i=entry_path_len+1, #path do - entry_path = lib.dirname(entry_path) + entry_path = dirname(entry_path) end local old = ietf_softwire_getter(entry_path)(config) -- Now figure out what the new entry should look like. @@ -469,8 +470,8 @@ local function ietf_softwire_translator () end function ret.remove_config(native_config, path) local ietf_binding_table_path = - '/softwire-config/binding/br-instances/br-instance[id=1]/binding-table' - if (lib.dirname(path) ~= ietf_binding_table_path or + '/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table' + if (dirname(path) ~= ietf_binding_table_path or path:sub(-1) ~= ']') then error('unsupported path: '..path) end @@ -492,7 +493,7 @@ local function ietf_softwire_translator () if (verb == 'remove' and path:match('^/softwire%-config/binding%-table/softwire')) then -- Remove a softwire. - local value = snabb_softwire_getter(path)(config) + local value = snabb_softwire_getter(path)(native_config) local br = cached_config.softwire_config.binding.br for _,instance in cltable.pairs(br.br_instances.br_instance) do local grammar = get_softwire_grammar() diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index 50d9dffaa4..cb656643d5 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -132,7 +132,7 @@ end function prepare_table_lookup(keys, ctype, query) local static_key = ctype and datalib.typeof(ctype)() or {} for k,_ in pairs(query) do - if not keys[k] then error("'"..key_name.."' is not a table key") end + if not keys[k] then error("'"..k.."' is not a table key") end end for k,grammar in pairs(keys) do local v = query[k] or grammar.default From 143aa98d6c034299e2d63b8e5fab15d01170c09f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 10:32:55 +0100 Subject: [PATCH 377/631] Fix default hash function bug in ctable Zomg people! I was hashing trash memory after the allocation, due to a boneheaded confusion between zero-based and one-based indexes and inclusive vs exclusive end bounds in FFI versus native Lua data. The tests did exercise this case but it happened not to matter. FML. --- src/lib/ctable.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 39259c117c..cbc44e85f2 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -14,6 +14,7 @@ CTable = {} LookupStreamer = {} local HASH_MAX = 0xFFFFFFFF +local uint8_ptr_t = ffi.typeof('uint8_t*') local uint16_ptr_t = ffi.typeof('uint16_t*') local uint32_ptr_t = ffi.typeof('uint32_t*') local uint64_ptr_t = ffi.typeof('uint64_t*') @@ -559,8 +560,8 @@ function compute_hash_fn(ctype) hash_fns_by_size[size] = function(key) local h = 0 local words = cast(uint32_ptr_t, key) - local bytes = cast('uint8_t*', key) - for i=0,size/4 do h = hash_32(bxor(h, words[i])) end + local bytes = cast(uint8_ptr_t, key) + for i=1,size/4 do h = hash_32(bxor(h, words[i-1])) end for i=1,size%4 do h = hash_32(bxor(h, bytes[size-i])) end return h end From cb14656dc31bd04345d48bdf6c13b28ad41450eb Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 10:39:59 +0100 Subject: [PATCH 378/631] Add missing local in ietf softwire support --- src/apps/config/support/snabb-softwire-v1.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 0c255569cc..cfeab28557 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -471,6 +471,7 @@ local function ietf_softwire_translator () function ret.remove_config(native_config, path) local ietf_binding_table_path = '/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table' + local softwire_path = '/softwire-config/binding-table/softwire' if (dirname(path) ~= ietf_binding_table_path or path:sub(-1) ~= ']') then error('unsupported path: '..path) From b4b0a7208aaa8f5772f6b35aba140a33bcefdd8e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 10:55:49 +0100 Subject: [PATCH 379/631] Fix fast-path bugs in ietf-softwire translator pre_update function We were never hitting the fast paths because the verb and path arguments were swapped. Also we were never detecting that a br-address was already in the set, because of slight differences between reference and pointer types (!) -- the br-addresses from the array were reference types, while the br-addresses from the update were pointers. --- src/apps/config/support/snabb-softwire-v1.lua | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index cfeab28557..37238c02c1 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -204,6 +204,12 @@ local function serialize_binding_table(bt) return printer(bt, yang.string_output_file()) end +local uint64_ptr_t = ffi.typeof('uint64_t*') +function ipv6_equals(a, b) + local x, y = ffi.cast(uint64_ptr_t, a), ffi.cast(uint64_ptr_t, b) + return x[0] == y[0] and x[1] == y[1] +end + local function ietf_softwire_translator () local ret = {} local cached_config @@ -340,7 +346,7 @@ local function ietf_softwire_translator () local new_br local bt = native_config.softwire_config.binding_table for i,br in ipairs(bt.br_address) do - if equal(br, new.br_ipv6_addr) then + if ipv6_equals(br, new.br_ipv6_addr) then new_br = i - 1; break end end @@ -435,7 +441,7 @@ local function ietf_softwire_translator () for _,new_br_address in ipairs(new_bt.br_address) do local idx for i,old_br_address in ipairs(old_bt.br_address) do - if equal(old_br_address, new_br_address) then + if ipv6_equals(old_br_address, new_br_address) then idx = i - 1 -- zero-based indexes, fml break end @@ -485,7 +491,7 @@ local function ietf_softwire_translator () return {{'remove', {schema='snabb-softwire-v1', path=softwire_path..query}}} end - function ret.pre_update(native_config, path, verb, data) + function ret.pre_update(native_config, verb, path, data) -- Given the notification that the native config is about to be -- updated, make our cached config follow along if possible (and -- if we have one). Otherwise throw away our cached config; we'll @@ -497,10 +503,10 @@ local function ietf_softwire_translator () local value = snabb_softwire_getter(path)(native_config) local br = cached_config.softwire_config.binding.br for _,instance in cltable.pairs(br.br_instances.br_instance) do - local grammar = get_softwire_grammar() + local grammar = get_ietf_softwire_grammar() local key = path_mod.prepare_table_lookup( - grammar.keys, grammar.key_ctype, - {['binding-ipv6info']=value.b4_ipv6}) + grammar.keys, grammar.key_ctype, {['binding-ipv6info']='::'}) + key.binding_ipv6info = value.b4_ipv6 assert(instance.binding_table.binding_entry[key] ~= nil) instance.binding_table.binding_entry[key] = nil end From 7b464b17a268330eda59e3a3234c947f5e0cdfd4 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 11:40:05 +0100 Subject: [PATCH 380/631] Fix ietf-softwire updates to individual binding table entries --- src/apps/config/support/snabb-softwire-v1.lua | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 37238c02c1..7a0b7392cc 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -302,7 +302,7 @@ local function ietf_softwire_translator () -- An update to an existing entry. First, get the existing entry. local config = ret.get_config(native_config) local entry_path = path_str - local entry_path_len = #br_instance_paths + #br_paths + local entry_path_len = #br_instance_paths + #bt_paths for i=entry_path_len+1, #path do entry_path = dirname(entry_path) end @@ -314,26 +314,26 @@ local function ietf_softwire_translator () else new = { port_set = { - psid_offset = entry.port_set.psid_offset, - psid_len = entry.port_set.psid_len, - psid = entry.port_set.psid + psid_offset = old.port_set.psid_offset, + psid_len = old.port_set.psid_len, + psid = old.port_set.psid }, - binding_ipv4_addr = entry.binding_ipv4_addr, - br_ipv6_addr = entry.br_ipv6_addr + binding_ipv4_addr = old.binding_ipv4_addr, + br_ipv6_addr = old.br_ipv6_addr } - if path[#entry_path_len + 1] == 'port-set' then - if #path == #entry_path_len + 1 then + if path[entry_path_len + 1].name == 'port-set' then + if #path == entry_path_len + 1 then new.port_set = arg else - local k = data.normalize_id(path[#path]) + local k = data.normalize_id(path[#path].name) new.port_set[k] = arg end - elseif path[#path] == 'binding-ipv4-addr' then + elseif path[#path].name == 'binding-ipv4-addr' then new.binding_ipv4_addr = arg - elseif path[#path] == 'br-ipv6-addr' then + elseif path[#path].name == 'br-ipv6-addr' then new.br_ipv6_addr = arg else - error('bad path element: '..path[#path]) + error('bad path element: '..path[#path].name) end end -- Apply changes. Start by ensuring that there's a br-address @@ -392,13 +392,17 @@ local function ietf_softwire_translator () return string.format('[ipv4=%s][psid=%s]', ipv4_ntop(ipv4), psid) end local old_query = q(old.binding_ipv4_addr, old.port_set.psid) + -- FIXME: This remove will succeed but the add could fail if + -- there's already a softwire with this IPv4 and PSID. We need + -- to add a check here that the IPv4/PSID is not present in the + -- binding table. table.insert(updates, {'remove', {schema='snabb-softwire-v1', path=softwire_path..old_query}}) local config_str = string.format( '{ ipv4 %s; psid %s; br %s; b4-ipv6 %s; }', ipv4_ntop(new.binding_ipv4_addr), new.port_set.psid, - new_br, path[#entry_path_len].query['binding-ipv6info']) + new_br, path[entry_path_len].query['binding-ipv6info']) table.insert(updates, {'add', {schema='snabb-softwire-v1', path=softwire_path, From f0f72338939b64b3ec487022c61072ffa1512482 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 12:06:21 +0100 Subject: [PATCH 381/631] Switch to 1-based indexing in snabb-softwire-v1 This changes the "br" field in softwires to be 1-indexed, and adds a "3.0.1.1" configuration migrator. --- src/apps/config/support/snabb-softwire-v1.lua | 15 +++++---- src/apps/lwaftr/binding_table.lua | 26 +++++++-------- src/lib/yang/snabb-softwire-v1.yang | 4 +-- .../lwaftr/migrate_configuration/README | 3 ++ .../migrate_configuration.lua | 32 +++++++++++++++++-- .../lwaftr/tests/data/big_mtu_no_icmp.conf | 4 +-- .../lwaftr/tests/data/icmp_on_fail.conf | 4 +-- src/program/lwaftr/tests/data/no_hairpin.conf | 4 +-- src/program/lwaftr/tests/data/no_icmp.conf | 4 +-- .../lwaftr/tests/data/no_icmp_maxfrags1.conf | 4 +-- .../data/no_icmp_with_filters_accept.conf | 4 +-- .../no_icmp_with_filters_and_vlan_accept.conf | 4 +-- .../no_icmp_with_filters_and_vlan_drop.conf | 4 +-- .../tests/data/no_icmp_with_filters_drop.conf | 4 +-- .../tests/data/small_ipv4_mtu_icmp.conf | 4 +-- .../tests/data/small_ipv6_mtu_no_icmp.conf | 4 +-- .../data/small_ipv6_mtu_no_icmp_allow.conf | 4 +-- .../small_ipv6_mtu_no_icmp_vlan_allow.conf | 4 +-- .../lwaftr/tests/data/tunnel_icmp.conf | 4 +-- .../tests/data/tunnel_icmp_without_mac4.conf | 4 +-- .../tests/data/tunnel_icmp_withoutmac.conf | 4 +-- src/program/lwaftr/tests/data/vlan.conf | 4 +-- .../tests/data/vlan/big_mtu_no_icmp.conf | 4 +-- .../lwaftr/tests/data/vlan/icmp_on_fail.conf | 4 +-- .../lwaftr/tests/data/vlan/no_hairpin.conf | 4 +-- .../lwaftr/tests/data/vlan/no_icmp.conf | 4 +-- .../tests/data/vlan/no_icmp_maxfrags1.conf | 4 +-- .../vlan/no_icmp_with_filters_accept.conf | 4 +-- .../data/vlan/no_icmp_with_filters_drop.conf | 4 +-- .../tests/data/vlan/small_ipv4_mtu_icmp.conf | 4 +-- .../data/vlan/small_ipv6_mtu_no_icmp.conf | 4 +-- .../vlan/small_ipv6_mtu_no_icmp_allow.conf | 4 +-- .../lwaftr/tests/data/vlan/tunnel_icmp.conf | 4 +-- .../data/vlan/tunnel_icmp_without_mac4.conf | 4 +-- .../data/vlan/tunnel_icmp_withoutmac.conf | 4 +-- src/program/lwaftr/tests/data/vlan/vlan.conf | 4 +-- 36 files changed, 117 insertions(+), 87 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 7a0b7392cc..3428b07160 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -143,7 +143,7 @@ local function ietf_binding_table_from_native(bt) psid_len = psid_map.psid_length, psid = entry.key.psid }, - br_ipv6_addr = bt.br_address[entry.value.br+1], + br_ipv6_addr = bt.br_address[entry.value.br], } ret[k] = v end @@ -182,8 +182,8 @@ local function native_binding_table_from_ietf(ietf) local br_address_key = ipv6:ntop(v.br_ipv6_addr) local br = br_address_by_ipv6[br_address_key] if not br then - br = #br_address table.insert(br_address, v.br_ipv6_addr) + br = #br_address br_address_by_ipv6[br_address_key] = br end local psid_key = psid_map_key_t({addr=v.binding_ipv4_addr}) @@ -347,11 +347,11 @@ local function ietf_softwire_translator () local bt = native_config.softwire_config.binding_table for i,br in ipairs(bt.br_address) do if ipv6_equals(br, new.br_ipv6_addr) then - new_br = i - 1; break + new_br = i; break end end if new_br == nil then - new_br = #bt.br_address + new_br = #bt.br_address + 1 table.insert(updates, {'add', {schema='snabb-softwire-v1', path=br_address_path, @@ -446,12 +446,13 @@ local function ietf_softwire_translator () local idx for i,old_br_address in ipairs(old_bt.br_address) do if ipv6_equals(old_br_address, new_br_address) then - idx = i - 1 -- zero-based indexes, fml + idx = i break end end if not idx then - idx, br_address_count = br_address_count, br_address_count + 1 + br_address_count = br_address_count + 1 + idx = br_address_count table.insert(updates, {'add', {schema='snabb-softwire-v1', path=br_address_path, @@ -469,7 +470,7 @@ local function ietf_softwire_translator () local config_str = string.format( '{ ipv4 %s; psid %s; br %s; b4-ipv6 %s; }', ipv4_ntop(entry.key.ipv4), entry.key.psid, - br_address_map[entry.value.br + 1], ipv6:ntop(entry.value.b4_ipv6)) + br_address_map[entry.value.br], ipv6:ntop(entry.value.b4_ipv6)) table.insert(additions, config_str) end table.insert(updates, diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index a6958c06d2..99a252b3cc 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -198,7 +198,7 @@ function BindingTable:lookup_psid(ipv4, port) end function BindingTable:get_br_address(i) - return self.br_addresses[i+1] + return self.br_addresses[i] end -- Iterate over the set of IPv4 addresses managed by a binding @@ -244,7 +244,7 @@ end -- Each entry is a pointer with two members, "key" and "value". They -- key is a softwire_key_t and has "ipv4" and "psid" members. The value -- is a softwire_value_t and has "br" and "b4_ipv6" members. The br is --- a zero-based index into the br_addresses array, and b4_ipv6 is a +-- a one-based index into the br_addresses array, and b4_ipv6 is a -- uint8_t[16]. function BindingTable:iterate_softwires() return self.softwires:iterate() @@ -286,7 +286,7 @@ function selftest() br-address 8:9:a:b:c:d:e:f; br-address 1E:1:1:1:1:1:1:af; br-address 1E:2:2:2:2:2:2:af; - softwire { ipv4 178.79.150.233; psid 80; b4-ipv6 127:2:3:4:5:6:7:128; br 0; } + softwire { ipv4 178.79.150.233; psid 80; b4-ipv6 127:2:3:4:5:6:7:128; br 1; } softwire { ipv4 178.79.150.233; psid 2300; b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; psid 2700; b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.233; psid 4660; b4-ipv6 127:11:12:13:14:15:16:128; } @@ -295,8 +295,8 @@ function selftest() softwire { ipv4 178.79.150.233; psid 54192; b4-ipv6 127:11:12:13:14:15:16:128; } softwire { ipv4 178.79.150.15; psid 0; b4-ipv6 127:22:33:44:55:66:77:128; } softwire { ipv4 178.79.150.15; psid 1; b4-ipv6 127:22:33:44:55:66:77:128;} - softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; br 1; } - softwire { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; br 2; } + softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; br 2; } + softwire { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; br 3; } ]]) local ipv4_pton = require('lib.yang.util').ipv4_pton @@ -309,18 +309,18 @@ function selftest() assert(ffi.C.memcmp(ipv6_protocol:pton(ipv6), val.b4_ipv6, 16) == 0) assert(val.br == br) end - assert_lookup('178.79.150.233', 80, '127:2:3:4:5:6:7:128', 0) + assert_lookup('178.79.150.233', 80, '127:2:3:4:5:6:7:128', 1) assert(lookup('178.79.150.233', 79) == nil) assert(lookup('178.79.150.233', 81) == nil) - assert_lookup('178.79.150.15', 80, '127:22:33:44:55:66:77:128', 0) - assert_lookup('178.79.150.15', 4095, '127:22:33:44:55:66:77:128', 0) - assert_lookup('178.79.150.15', 4096, '127:22:33:44:55:66:77:128', 0) - assert_lookup('178.79.150.15', 8191, '127:22:33:44:55:66:77:128', 0) + assert_lookup('178.79.150.15', 80, '127:22:33:44:55:66:77:128', 1) + assert_lookup('178.79.150.15', 4095, '127:22:33:44:55:66:77:128', 1) + assert_lookup('178.79.150.15', 4096, '127:22:33:44:55:66:77:128', 1) + assert_lookup('178.79.150.15', 8191, '127:22:33:44:55:66:77:128', 1) assert(lookup('178.79.150.15', 8192) == nil) - assert_lookup('178.79.150.2', 7850, '127:24:35:46:57:68:79:128', 1) + assert_lookup('178.79.150.2', 7850, '127:24:35:46:57:68:79:128', 2) assert(lookup('178.79.150.3', 4095) == nil) - assert_lookup('178.79.150.3', 4096, '127:14:25:36:47:58:69:128', 2) - assert_lookup('178.79.150.3', 5119, '127:14:25:36:47:58:69:128', 2) + assert_lookup('178.79.150.3', 4096, '127:14:25:36:47:58:69:128', 3) + assert_lookup('178.79.150.3', 5119, '127:14:25:36:47:58:69:128', 3) assert(lookup('178.79.150.3', 5120) == nil) assert(lookup('178.79.150.4', 7850) == nil) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index bdf9a07029..9669e74f7a 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -352,10 +352,10 @@ module snabb-softwire-v1 { leaf br { type uint32; - default 0; + default 1; description "The B4-facing address of the lwAFTR for this softwire, as - a zero-based index into br-addresses."; + a one-based index into br-addresses."; } leaf b4-ipv6 { diff --git a/src/program/lwaftr/migrate_configuration/README b/src/program/lwaftr/migrate_configuration/README index a94efe8a8a..015d3561f5 100644 --- a/src/program/lwaftr/migrate_configuration/README +++ b/src/program/lwaftr/migrate_configuration/README @@ -14,6 +14,9 @@ configuration. Available VERSION values are: lwAFTR versions where "container" nodes in schemas are missing corresponding nodes in the data unless "presence true" is specified. + 3.0.1.1 + lwAFTR development snapshot where "br" fields of softwires were + 0-based instead of 1-based. The default version is "legacy". diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 0edd525a57..d2062ecfaf 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -235,7 +235,7 @@ local function parse_softwires(parser, psid_map, br_address_count) while not parser:check('}') do local entry = parser:parse_property_list(softwire_spec, '{', '}') key.ipv4, key.psid = entry.ipv4, entry.psid - value.br, value.b4_ipv6 = entry.aftr, entry.b4 + value.br, value.b4_ipv6 = entry.aftr + 1, entry.b4 local success = pcall(map.add, map, key, value) if not success then parser:error('duplicate softwire for ipv4=%s, psid=%d', @@ -349,13 +349,39 @@ local function migrate_legacy(stream) return migrate_conf(conf) end +local function increment_br(conf) + for entry in conf.softwire_config.binding_table.softwire:iterate() do + -- Sadly it's not easy to make an updater that always works for + -- the indexing change, because changing the default from 0 to 1 + -- makes it ambiguous whether a "br" value of 1 comes from the new + -- default, or was present as such in the old configuration. Sad. + if entry.value.br ~= 1 then + entry.value.br = entry.value.br + 1 + end + end + if #conf.softwire_config.binding_table.br_address > 1 then + io.stderr:write('Migrator unable to tell whether br=1 entries are '.. + 'due to new default or old setting; manual '.. + 'verification needed.\n') + io.stderr:flush() + end + return conf +end + local function migrate_3_0_1(conf_file) local data = require('lib.yang.data') local str = "softwire-config {\n"..io.open(conf_file, 'r'):read('*a').."\n}" - return data.load_data_for_schema_by_name('snabb-softwire-v1', str, conf_file) + return increment_br(data.load_data_for_schema_by_name( + 'snabb-softwire-v1', str, conf_file)) +end + +local function migrate_3_0_1bis(conf_file) + return increment_br(yang.load_configuration( + conf_file, {schema_name='snabb-softwire-v1'})) end -local migrators = { legacy = migrate_legacy, ['3.0.1'] = migrate_3_0_1 } +local migrators = { legacy = migrate_legacy, ['3.0.1'] = migrate_3_0_1, + ['3.0.1.1'] = migrate_3_0_1bis } function run(args) local conf_file, version = parse_args(args) local migrate = migrators[version] diff --git a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf index f6cbceff2d..6eba452354 100644 --- a/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/big_mtu_no_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/icmp_on_fail.conf b/src/program/lwaftr/tests/data/icmp_on_fail.conf index ae41560560..6ff06f5a80 100644 --- a/src/program/lwaftr/tests/data/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/icmp_on_fail.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/no_hairpin.conf b/src/program/lwaftr/tests/data/no_hairpin.conf index d9df3aeac2..f26f9d70c7 100644 --- a/src/program/lwaftr/tests/data/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/no_hairpin.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/no_icmp.conf b/src/program/lwaftr/tests/data/no_icmp.conf index 2b21e45d59..979aeb3747 100644 --- a/src/program/lwaftr/tests/data/no_icmp.conf +++ b/src/program/lwaftr/tests/data/no_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf index b42ef58b8f..989c368aa0 100644 --- a/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/no_icmp_maxfrags1.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf index d78af5683b..ee519ec99e 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_accept.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf index d97d9e077f..28be24b711 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_accept.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf index 584afa9ea2..ef27e919d9 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_and_vlan_drop.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf index 2e63cadb82..95dad9f273 100644 --- a/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/no_icmp_with_filters_drop.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf index f46104852c..518bbb8461 100644 --- a/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv4_mtu_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf index a7df23b0ee..51126e45d6 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf index 0e65ddd1c4..1efc6045ae 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_allow.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf index 775259a935..25a4846ca4 100644 --- a/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf +++ b/src/program/lwaftr/tests/data/small_ipv6_mtu_no_icmp_vlan_allow.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/tunnel_icmp.conf b/src/program/lwaftr/tests/data/tunnel_icmp.conf index 15b189b341..af65132b26 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf index 2751c2ce51..2f8cdc5503 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_without_mac4.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf index 0ce5c32a95..7b7d481acc 100644 --- a/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/tunnel_icmp_withoutmac.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan.conf b/src/program/lwaftr/tests/data/vlan.conf index f05fe81a61..8f8255aafc 100644 --- a/src/program/lwaftr/tests/data/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf index 3ffd81afa5..080802a293 100644 --- a/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/big_mtu_no_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf index e89cfb6a04..91957c73a2 100644 --- a/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf +++ b/src/program/lwaftr/tests/data/vlan/icmp_on_fail.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf index 1a31815ad9..171c4ba983 100644 --- a/src/program/lwaftr/tests/data/vlan/no_hairpin.conf +++ b/src/program/lwaftr/tests/data/vlan/no_hairpin.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp.conf b/src/program/lwaftr/tests/data/vlan/no_icmp.conf index ba864dd4c2..4a3750b486 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf index 6a2bd96a8a..d983df6fa0 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_maxfrags1.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf index d97d9e077f..28be24b711 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_accept.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf index 584afa9ea2..ef27e919d9 100644 --- a/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf +++ b/src/program/lwaftr/tests/data/vlan/no_icmp_with_filters_drop.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf index 54fdfe5699..e4a97b6384 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv4_mtu_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf index f42826c924..097d2ac8ed 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf index 775259a935..25a4846ca4 100644 --- a/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf +++ b/src/program/lwaftr/tests/data/vlan/small_ipv6_mtu_no_icmp_allow.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf index a3850cb57f..ea417b89a6 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf index 97def6e88b..288262f4fd 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_without_mac4.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf index 2b67e14617..2e3aab8c5c 100644 --- a/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf +++ b/src/program/lwaftr/tests/data/vlan/tunnel_icmp_withoutmac.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; diff --git a/src/program/lwaftr/tests/data/vlan/vlan.conf b/src/program/lwaftr/tests/data/vlan/vlan.conf index f05fe81a61..8f8255aafc 100644 --- a/src/program/lwaftr/tests/data/vlan/vlan.conf +++ b/src/program/lwaftr/tests/data/vlan/vlan.conf @@ -52,13 +52,13 @@ softwire-config { ipv4 178.79.150.3; psid 4; b4-ipv6 127:14:25:36:47:58:69:128; - br 2; + br 3; } softwire { ipv4 178.79.150.2; psid 7850; b4-ipv6 127:24:35:46:57:68:79:128; - br 1; + br 2; } softwire { ipv4 178.79.150.233; From 758e9dc9eab3d20b552d87e820518d0dd07515e1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 13:49:13 +0100 Subject: [PATCH 382/631] Add lwAFTR 3.1.0 changelog. --- src/program/lwaftr/doc/CHANGELOG.md | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index ee98fa0c56..d66b45d103 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,48 @@ # Change Log +## [3.1.0] - 2016-12-06 + +Adding "ietf-softwire" support, process separation between control and +the data plane, and some configuration file changes. + + * Passing --reconfigurable to "snabb lwaftr run" now forks off a + dedicated data plane child process. This removes the overhead of + --reconfigurable that was present in previous releases. + + * Add support for ietf-softwire. Pass the "-s ietf-softwire" to "snabb + config" invocations to use this schema. + + * Add support for fast binding-table updates. This is the first + version since the YANG migration that can make fast updates to + individual binding-table entries without causing the whole table to + reload, via "snabb config add + /softwire-config/binding-table/softwire". See "snabb config" + documentation for more on how to use "snabb config add" and "snabb + config remove". + + * Add support for named lwAFTR instances. Pass "--name foo" to the + "snabb lwaftr run" command to have it claim a name on a machine. + "snabb config" can identify the remote Snabb instance by name, which + is often much more convenient than using the instance's PID. + + * Final tweaks to the YANG schema before deployment -- now the + binding-table section is inside softwire-config, and the + configuration file format is now enclosed in "softwire-config {...}". + It used to be that only YANG "container" nodes which had "presence + true;" would have corresponding data nodes; this was a mistake. The + new mapping where every container node from the YANG schema appears + in the data more closely follows the YANG standard XML mapping that + the XPath expressions are designed to operate over. + + Additionally, the "br" leaf inside "snabb-softwire-v1" lists is now a + 1-based index into the "br-address" leaf-list instead of a zero-based + index. + + The "snabb lwaftr migrate-configation --from=3.0.1" command can + migrate your 3.0.1 configuration files to the new format. See "snabb + lwaftr migrate-configuration --help" for more details. The default + "--from" version is "legacy", meaning pre-3.0 lwAFTR configurations. + ## [3.0.1] - 2016-11-28 A release to finish "snabb config" features. From e8fce9cd0812d7b9c9d4473a4984a218f0834846 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 6 Dec 2016 15:05:23 +0100 Subject: [PATCH 383/631] Fixes undefined "otps" variable. This was a typo that wasn't caught in testing unfortunately. This is in setting up the name for the lwaftr in the "run" command. --- src/program/lwaftr/run/run.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 9ebe9ea3bf..6b3f79f261 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -147,7 +147,7 @@ function run(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local use_splitter = requires_splitter(opts, conf) - if opts.name then engine.claim_name(otps.name) end + if opts.name then engine.claim_name(opts.name) end local c = config.new() local setup_fn, setup_args From cbf74002e677d94c0381027684173cf7c2118887 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 6 Dec 2016 14:42:52 +0000 Subject: [PATCH 384/631] Store pid as a string --- src/program/top/top.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index c464bb32d2..01f3abd1fa 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -39,7 +39,7 @@ function select_snabb_instance (pid) for _, name in ipairs(shm.children("/")) do -- This could fail as the name could be for example "by-name" local p = tonumber(name) - if p and p ~= my_pid then table.insert(pids, p) end + if p and p ~= my_pid then table.insert(pids, name) end end return pids end From 8a147ca09fb1134e748f12eb17631b9743856361 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Tue, 6 Dec 2016 16:05:59 +0100 Subject: [PATCH 385/631] Make lwaftr run --reconfigurable uniprocess --- src/program/lwaftr/bench/bench.lua | 4 ++-- src/program/lwaftr/run/run.lua | 13 ++----------- src/program/lwaftr/selftest.sh | 3 ++- src/program/lwaftr/setup.lua | 17 ++++++++++++++++- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 49df4f048a..b8f025d67d 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -56,8 +56,8 @@ function run(args) local graph = config.new() if opts.reconfigurable then - setup.reconfigurable(setup.load_bench, graph, conf, - inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + setup.reconfigurable_multi(setup.load_bench, graph, conf, + inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') else setup.load_bench(graph, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') end diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 6b3f79f261..c952a242f2 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -43,16 +43,7 @@ function parse_args(args) if not cpu or cpu ~= math.floor(cpu) or cpu < 0 then fatal("Invalid cpu number: "..arg) end - - if opts.reconfigurable then - S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) - local wanted_node = numa.cpu_get_numa_node(cpu) - numa.bind_to_numa_node(wanted_node) - print("Bound to numa node:", wanted_node) - else - print("Bound to CPU:", cpu) - numa.bind_to_cpu(cpu) - end + numa.bind_to_cpu(cpu) end handlers['real-time'] = function(arg) if not S.sched_setscheduler(0, "fifo", 1) then @@ -163,7 +154,7 @@ function run(args) setup_fn, setup_args = setup.load_phy, { 'inetNic', v4, 'b4sideNic', v6 } end if opts.reconfigurable then - setup.reconfigurable(setup_fn, c, conf, unpack(setup_args)) + setup.reconfigurable_uni(setup_fn, c, conf, unpack(setup_args)) else setup_fn(c, conf, unpack(setup_args)) end diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh index 7133693df3..e454e30a28 100755 --- a/src/program/lwaftr/selftest.sh +++ b/src/program/lwaftr/selftest.sh @@ -37,6 +37,7 @@ echo "Testing snabb lwaftr run" sudo ./snabb lwaftr run -D 0.1 --conf ${TDIR}/icmp_on_fail.conf \ --on-a-stick "$SNABB_PCI0" +# Keep the following test at 1 second; 0.1 can hide problems, empirically echo "Testing snabb lwaftr run --reconfigurable" -sudo ./snabb lwaftr run -D 0.1 --reconfigurable \ +sudo ./snabb lwaftr run -D 1 --reconfigurable \ --conf ${TDIR}/icmp_on_fail.conf --on-a-stick "$SNABB_PCI0" diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 956f11cb88..36a15e00ca 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -467,7 +467,7 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) link_sink(c, unpack(sinks)) end -function reconfigurable(f, graph, conf, ...) +function reconfigurable_multi(f, graph, conf, ...) local args = {...} local function setup_fn(conf) local graph = config.new() @@ -499,3 +499,18 @@ function reconfigurable(f, graph, conf, ...) follower_pids = { follower_pid }, schema_name = 'snabb-softwire-v1'}) end + +function reconfigurable_uni(f, graph, conf, ...) + local args = {...} + local function setup_fn(conf) + local graph = config.new() + f(graph, conf, unpack(args)) + return graph + end + + config.app(graph, 'leader', leader.Leader, + { setup_fn = setup_fn, initial_configuration = conf, + follower_pids = { S.getpid() }, + schema_name = 'snabb-softwire-v1'}) + config.app(graph, "follower", follower.Follower, {}) +end From c378451382a26009a882bb116c38840250c70eda Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 6 Dec 2016 16:27:12 +0100 Subject: [PATCH 386/631] Snabb lwAFTR v3.1.1 change log entry --- src/program/lwaftr/doc/CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index d66b45d103..999002f012 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,16 @@ # Change Log +## [3.1.1] - 2016-12-06 + +A hotfix to work around bugs in multiprocess support when using Intel +NICs. + + * Passing --reconfigurable to "snabb lwaftr run" now just uses a single + process while we sort out multiprocess issues. + + * Fixed "snabb lwaftr query" and "snabb top", broken during + refactoring. + ## [3.1.0] - 2016-12-06 Adding "ietf-softwire" support, process separation between control and @@ -33,7 +44,7 @@ the data plane, and some configuration file changes. new mapping where every container node from the YANG schema appears in the data more closely follows the YANG standard XML mapping that the XPath expressions are designed to operate over. - + Additionally, the "br" leaf inside "snabb-softwire-v1" lists is now a 1-based index into the "br-address" leaf-list instead of a zero-based index. From 037aefad42059d6ac9c0c40465acbd7e6c2c360c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 08:52:01 +0100 Subject: [PATCH 387/631] Incrementally lex and parse data Instead of building an AST, then parsing that AST, incrementally parse as we go. This should prevent the creation of large intermediate data structures when parsing 100-MB files that could push us over LuaJIT's 1G limit. --- src/lib/yang/data.lua | 135 ++++++++++++++++++++++++---------------- src/lib/yang/parser.lua | 2 +- 2 files changed, 81 insertions(+), 56 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 17baaad85a..fef68253cf 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -171,33 +171,24 @@ local function value_parser(typ) end end -local function assert_scalar(node, keyword, opts) - assert(node.argument or (opts and opts.allow_empty_argument), - 'missing argument for "'..keyword..'"') - assert(not node.statements, 'unexpected sub-parameters for "'..keyword..'"') -end - -local function assert_compound(node, keyword) - assert(not node.argument, 'argument unexpected for "'..keyword..'"') - assert(node.statements, - 'missing brace-delimited sub-parameters for "'..keyword..'"') -end - -local function assert_not_duplicate(out, keyword) - assert(not out, 'duplicate parameter: '..keyword) -end - local function struct_parser(keyword, members, ctype) local function init() return nil end - local function parse1(node) - assert_compound(node, keyword) + local function parse1(P) local ret = {} for k,sub in pairs(members) do ret[normalize_id(k)] = sub.init() end - for _,node in ipairs(node.statements) do - local sub = assert(members[node.keyword], - 'unrecognized parameter: '..node.keyword) - local id = normalize_id(node.keyword) - ret[id] = sub.parse(node, ret[id]) + P:skip_whitespace() + P:consume("{") + P:skip_whitespace() + while not P:check("}") do + local k = P:parse_identifier() + if k == '' then P:error("Expected a keyword") end + -- Scalar/array parser responsible for requiring whitespace + -- after keyword. Struct/table don't need it as they have + -- braces. + local sub = assert(members[k], 'unrecognized parameter: '..k) + local id = normalize_id(k) + ret[id] = sub.parse(P, ret[id]) + P:skip_whitespace() end for k,sub in pairs(members) do local id = normalize_id(k) @@ -205,9 +196,9 @@ local function struct_parser(keyword, members, ctype) end return ret end - local function parse(node, out) - assert_not_duplicate(out, keyword) - return parse1(node) + local function parse(P, out) + if out ~= nil then P:error('duplicate parameter: '..keyword) end + return parse1(P) end local struct_t = ctype and typeof(ctype) local function finish(out) @@ -220,12 +211,15 @@ end local function array_parser(keyword, element_type, ctype) local function init() return {} end local parsev = value_parser(element_type) - local function parse1(node) - assert_scalar(node, keyword) - return parsev(node.argument, keyword) - end - local function parse(node, out) - table.insert(out, parse1(node)) + local function parse1(P) + P:consume_whitespace() + local str = P:parse_string() + P:skip_whitespace() + P:consume(";") + return parsev(str, keyword) + end + local function parse(P, out) + table.insert(out, parse1(P)) return out end local elt_t = ctype and typeof(ctype) @@ -243,13 +237,19 @@ end local function scalar_parser(keyword, argument_type, default, mandatory) local function init() return nil end local parsev = value_parser(argument_type) - local function parse1(node) - assert_scalar(node, keyword, {allow_empty_argument=true}) - return parsev(node.argument, keyword) + local function parse1(P) + local maybe_str + if argument_type.primitive_type ~= 'empty' then + P:consume_whitespace() + maybe_str = P:parse_string() + end + P:skip_whitespace() + P:consume(";") + return parsev(maybe_str, keyword) end - local function parse(node, out) - assert_not_duplicate(out, keyword) - return parse1(node) + local function parse(P, out) + if out ~= nil then P:error('duplicate parameter: '..keyword) end + return parse1(P) end local function finish(out) if out ~= nil then return out end @@ -316,12 +316,11 @@ local function table_parser(keyword, keys, values, string_key, key_ctype, else function init() return ltable_builder() end end - local function parse1(node) - assert_compound(node, keyword) - return parser.finish(parser.parse(node, parser.init())) + local function parse1(P) + return parser.finish(parser.parse(P, parser.init())) end - local function parse(node, assoc) - local struct = parse1(node) + local function parse(P, assoc) + local struct = parse1(P) local key, value = {}, {} if key_t then key = key_t() end if value_t then value = value_t() end @@ -377,21 +376,41 @@ function data_parser_from_grammar(production) local top_parsers = {} function top_parsers.struct(production) - local parser = visit1('(top level)', production) + local struct_t = production.ctype and typeof(production.ctype) + local members = visitn(production.members) return function(str, filename) - local node = {statements=parser_mod.parse(str, filename)} - return parser.finish(parser.parse(node, parser.init())) + local P = parser_mod.Parser.new(str, filename) + local ret = {} + for k,sub in pairs(members) do ret[normalize_id(k)] = sub.init() end + while true do + P:skip_whitespace() + if P:is_eof() then break end + local k = P:parse_identifier() + if k == '' then P:error("Expected a keyword") end + local sub = assert(members[k], 'unrecognized parameter: '..k) + local id = normalize_id(k) + ret[id] = sub.parse(P, ret[id]) + end + for k,sub in pairs(members) do + local id = normalize_id(k) + ret[id] = sub.finish(ret[id]) + end + if struct_t then return struct_t(ret) else return ret end end end function top_parsers.sequence(production) local members = visitn(production.members) return function(str, filename) + local P = parser_mod.Parser.new(str, filename) local ret = {} - for _, node in ipairs(parser_mod.parse(str, filename)) do - local sub = assert(members[node.keyword], - 'unrecognized rpc: '..node.keyword) - local data = sub.finish(sub.parse(node, sub.init())) - table.insert(ret, {id=node.keyword, data=data}) + while true do + P:skip_whitespace() + if P:is_eof() then break end + local k = P:parse_identifier() + P:consume_whitespace() + local sub = assert(members[k], 'unrecognized rpc: '..k) + local data = sub.finish(sub.parse(P, sub.init())) + table.insert(ret, {id=k, data=data}) end return ret end @@ -399,9 +418,12 @@ function data_parser_from_grammar(production) function top_parsers.array(production) local parser = visit1('[bare array]', production) return function(str, filename) + local P = parser_mod.Parser.new(str, filename) local out = parser.init() - for _,v in ipairs(parser_mod.parse_strings(str, filename)) do - out = parser.parse({keyword='[bare array]', argument=v}, out) + while true do + P:skip_whitespace() + if P:is_eof() then break end + out = parser.parse(P, out) end return parser.finish(out) end @@ -409,9 +431,12 @@ function data_parser_from_grammar(production) function top_parsers.table(production) local parser = visit1('[bare table]', production) return function(str, filename) + local P = parser_mod.Parser.new(str, filename) local out = parser.init() - for _,v in ipairs(parser_mod.parse_statement_lists(str, filename)) do - out = parser.parse({keyword='[bare table]', statements=v}, out) + while true do + P:skip_whitespace() + if P:is_eof() then break end + out = parser.parse(P, out) end return parser.finish(out) end diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 97db2b7684..023e627af6 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -30,7 +30,7 @@ module(..., package.seeall) local lib = require('core.lib') -local Parser = {} +Parser = {} function Parser.new(str, filename) local ret = {pos=1, str=str, filename=filename, line=1, column=0, line_pos=1} ret = setmetatable(ret, {__index = Parser}) From f46f113bc036725d5ea45813694ec17363dcfea3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 08:42:45 +0000 Subject: [PATCH 388/631] Micro-optimizations to data parser. --- src/lib/ctable.lua | 3 ++- src/lib/yang/data.lua | 19 +++++++++++++------ src/lib/yang/util.lua | 17 +++++++++++------ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index cbc44e85f2..29aa0a54b4 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -219,7 +219,8 @@ function CTable:insert(hash, key, value, updates_allowed) local entries = self.entries local scale = self.scale - local start_index = hash_to_index(hash, self.scale) + -- local start_index = hash_to_index(hash, self.scale) + local start_index = floor(hash*self.scale + 0.5) local index = start_index while entries[index].hash < hash do diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index fef68253cf..0893e655bf 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -157,7 +157,7 @@ local function enum_validator(range, f) return f end local function value_parser(typ) local prim = typ.primitive_type local parse = assert(value.types[prim], prim).parse - local function validate(val) end + local validate validate = range_validator(typ.range, validate) validate = length_validator(typ.length, validate) validate = pattern_validator(typ.pattern, validate) @@ -166,16 +166,18 @@ local function value_parser(typ) -- TODO: union, require-instance. return function(str, k) local val = parse(str, k) - validate(val) + if validate then validate(val) end return val end end local function struct_parser(keyword, members, ctype) + local keys = {} + for k,v in pairs(members) do table.insert(keys, k) end local function init() return nil end local function parse1(P) local ret = {} - for k,sub in pairs(members) do ret[normalize_id(k)] = sub.init() end + for _,k in ipairs(keys) do ret[normalize_id(k)] = members[k].init() end P:skip_whitespace() P:consume("{") P:skip_whitespace() @@ -190,9 +192,9 @@ local function struct_parser(keyword, members, ctype) ret[id] = sub.parse(P, ret[id]) P:skip_whitespace() end - for k,sub in pairs(members) do + for _,k in ipairs(keys) do local id = normalize_id(k) - ret[id] = sub.finish(ret[id]) + ret[id] = members[k].finish(ret[id]) end return ret end @@ -262,7 +264,12 @@ end local function ctable_builder(key_t, value_t) local res = ctable.new({ key_type=key_t, value_type=value_t }) local builder = {} - function builder:add(key, value) res:add(key, value) end + local counter = 0 + function builder:add(key, value) + counter = counter + 1 + if counter % 1000 == 0 then print('ctable add', counter) end + res:add(key, value) + end function builder:finish() return res end return builder end diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 339f5247d7..ee414a807f 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -20,23 +20,28 @@ function tointeger(str, what, min, max) if str:match('^0x', start) then base, start = 16, start + 2 elseif str:match('^0', start) then base = 8 end str = str:lower() - local function check(test) - return assert(test, 'invalid numeric value for '..what..': '..str) + if start > str:len() then + error('invalid numeric value for '..what..': '..str) end - check(start <= str:len()) -- FIXME: check that res did not overflow the 64-bit number local res = ffi.C.strtoull(str:sub(start), nil, base) if is_negative then res = ffi.new('int64_t[1]', -1*res)[0] - check(res <= 0) + if res > 0 then + error('invalid numeric value for '..what..': '..str) + end if min then check(min <= 0 and min <= res) end else -- Only compare min and res if both are positive, otherwise if min -- is a negative int64_t then the comparison will treat it as a -- large uint64_t. - if min then check(min <= 0 or min <= res) end + if min and not (min <= 0 or min <= res) then + error('invalid numeric value for '..what..': '..str) + end + end + if max and res > max then + error('invalid numeric value for '..what..': '..str) end - if max then check(res <= max) end -- Only return Lua numbers for values within int32 + uint32 range. -- The 0 <= res check is needed because res might be a uint64, in -- which case comparing to a negative Lua number will cast that Lua From 70b5d73e6703ff9a6c5ee1b2e8339359efc965ea Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 08:57:17 +0000 Subject: [PATCH 389/631] Add a fast path for ctable insert. --- src/lib/ctable.lua | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 29aa0a54b4..9898fa3cac 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -223,6 +223,15 @@ function CTable:insert(hash, key, value, updates_allowed) local start_index = floor(hash*self.scale + 0.5) local index = start_index + -- Fast path. + if entries[index].hash == HASH_MAX and updates_allowed ~= 'required' then + self.occupancy = self.occupancy + 1 + entries[index].hash = hash + entries[index].key = key + entries[index].value = value + return index + end + while entries[index].hash < hash do index = index + 1 end From 7d8160e38f86f3bc3a03b3ad280bd76ecc42326a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 7 Dec 2016 10:41:53 +0100 Subject: [PATCH 390/631] Fix #622 Use correct PID when getting state This fixes a problem where `snabb config get-state` would fail with no state information being found. The problem was from when we had the leader and the lwaftr share a process. The fix was to iterate over the followers, it now adds all the counters together. --- src/apps/config/leader.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 3bd0fc60d0..53dcde7e93 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -518,8 +518,13 @@ function Leader:rpc_get_state (args) return self:foreign_rpc_get_state(args.schema, args.path) end local printer = path_printer_for_schema_by_name(self.schema_name, args.path) - local state = state.show_state(self.schema_name, S.getpid(), args.path) - return {state=printer(state, yang.string_output_file())} + local s = {} + for _, follower in pairs(self.followers) do + for k,v in pairs(state.show_state(self.schema_name, follower.pid, args.path)) do + s[k] = v + end + end + return {state=printer(s, yang.string_output_file())} end function Leader:handle (payload) From de2b55d5daa7e54fb992d298b646ee4b4ce60646 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 09:50:36 +0000 Subject: [PATCH 391/631] Use 40% occupancy as a maximum for data ctables. --- src/lib/ctable.lua | 4 +++- src/lib/yang/data.lua | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 9898fa3cac..87de710d6e 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -111,6 +111,7 @@ function new(params) ctab.hash_fn = params.hash_fn or compute_hash_fn(params.key_type) ctab.equal_fn = make_equal_fn(params.key_type) ctab.size = 0 + ctab.max_displacement = 0 ctab.occupancy = 0 ctab.max_occupancy_rate = params.max_occupancy_rate ctab.min_occupancy_rate = params.min_occupancy_rate @@ -149,6 +150,7 @@ function CTable:resize(size) assert(size >= (self.occupancy / self.max_occupancy_rate)) local old_entries = self.entries local old_size = self.size + local old_max_displacement = self.max_displacement -- Allocate double the requested number of entries to make sure there -- is sufficient displacement if all hashes map to the last bucket. @@ -161,7 +163,7 @@ function CTable:resize(size) self.occupancy_lo = floor(self.size * self.min_occupancy_rate) for i=0,self.size*2-1 do self.entries[i].hash = HASH_MAX end - for i=0,old_size*2-1 do + for i=0,old_size+old_max_displacement-1 do if old_entries[i].hash ~= HASH_MAX then self:insert(old_entries[i].hash, old_entries[i].key, old_entries[i].value) end diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 0893e655bf..6ba3c1cd97 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -262,12 +262,14 @@ local function scalar_parser(keyword, argument_type, default, mandatory) end local function ctable_builder(key_t, value_t) - local res = ctable.new({ key_type=key_t, value_type=value_t }) + local res = ctable.new({ key_type=key_t, value_type=value_t, + max_occupancy_rate = 0.4 }) local builder = {} - local counter = 0 + -- Uncomment for progress counters. + -- local counter = 0 function builder:add(key, value) - counter = counter + 1 - if counter % 1000 == 0 then print('ctable add', counter) end + -- counter = counter + 1 + -- if counter % 1000 == 0 then print('ctable add', counter) end res:add(key, value) end function builder:finish() return res end From 6d30d1df59317b557bb9beb356923f3ba17dea7d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 11:33:38 +0100 Subject: [PATCH 392/631] Inline another call to check in tointeger --- src/lib/yang/util.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index ee414a807f..91e8ff17c5 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -30,7 +30,9 @@ function tointeger(str, what, min, max) if res > 0 then error('invalid numeric value for '..what..': '..str) end - if min then check(min <= 0 and min <= res) end + if min and not (min <= 0 and min <= res) then + error('invalid numeric value for '..what..': '..str) + end else -- Only compare min and res if both are positive, otherwise if min -- is a negative int64_t then the comparison will treat it as a From a357e11994f1f0aef778b561fc787a042a760180 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 11:49:33 +0100 Subject: [PATCH 393/631] Fix bug in ctable hash-to-index mapping Before, we would sometimes map a hash beyond the size of a table. It didn't usually matter before as resizing always visited 2*size. Now that resize precisely visits only the active entries -- those within [0,size+max_displacement) -- now we see this bug. Caught by the cltable test suite, yay! --- src/lib/ctable.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 87de710d6e..fd13968f7f 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -35,7 +35,7 @@ end -- hash := [0,HASH_MAX); scale := size/HASH_MAX local function hash_to_index(hash, scale) - return floor(hash*scale + 0.5) + return floor(hash*scale) end local function make_equal_fn(key_type) @@ -222,7 +222,7 @@ function CTable:insert(hash, key, value, updates_allowed) local entries = self.entries local scale = self.scale -- local start_index = hash_to_index(hash, self.scale) - local start_index = floor(hash*self.scale + 0.5) + local start_index = floor(hash*self.scale) local index = start_index -- Fast path. From 7e7190d5a9509d50dd9a1c7c5da78fcd7d3a8a17 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Wed, 7 Dec 2016 12:56:27 +0100 Subject: [PATCH 394/631] Revert "Make lwaftr run --reconfigurable uniprocess" This reverts commit 8a147ca09fb1134e748f12eb17631b9743856361. This patch was always intended to be reverted as soon as possible. --- src/program/lwaftr/bench/bench.lua | 4 ++-- src/program/lwaftr/run/run.lua | 13 +++++++++++-- src/program/lwaftr/selftest.sh | 3 +-- src/program/lwaftr/setup.lua | 17 +---------------- 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index b8f025d67d..49df4f048a 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -56,8 +56,8 @@ function run(args) local graph = config.new() if opts.reconfigurable then - setup.reconfigurable_multi(setup.load_bench, graph, conf, - inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + setup.reconfigurable(setup.load_bench, graph, conf, + inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') else setup.load_bench(graph, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') end diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index c952a242f2..6b3f79f261 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -43,7 +43,16 @@ function parse_args(args) if not cpu or cpu ~= math.floor(cpu) or cpu < 0 then fatal("Invalid cpu number: "..arg) end - numa.bind_to_cpu(cpu) + + if opts.reconfigurable then + S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) + local wanted_node = numa.cpu_get_numa_node(cpu) + numa.bind_to_numa_node(wanted_node) + print("Bound to numa node:", wanted_node) + else + print("Bound to CPU:", cpu) + numa.bind_to_cpu(cpu) + end end handlers['real-time'] = function(arg) if not S.sched_setscheduler(0, "fifo", 1) then @@ -154,7 +163,7 @@ function run(args) setup_fn, setup_args = setup.load_phy, { 'inetNic', v4, 'b4sideNic', v6 } end if opts.reconfigurable then - setup.reconfigurable_uni(setup_fn, c, conf, unpack(setup_args)) + setup.reconfigurable(setup_fn, c, conf, unpack(setup_args)) else setup_fn(c, conf, unpack(setup_args)) end diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh index e454e30a28..7133693df3 100755 --- a/src/program/lwaftr/selftest.sh +++ b/src/program/lwaftr/selftest.sh @@ -37,7 +37,6 @@ echo "Testing snabb lwaftr run" sudo ./snabb lwaftr run -D 0.1 --conf ${TDIR}/icmp_on_fail.conf \ --on-a-stick "$SNABB_PCI0" -# Keep the following test at 1 second; 0.1 can hide problems, empirically echo "Testing snabb lwaftr run --reconfigurable" -sudo ./snabb lwaftr run -D 1 --reconfigurable \ +sudo ./snabb lwaftr run -D 0.1 --reconfigurable \ --conf ${TDIR}/icmp_on_fail.conf --on-a-stick "$SNABB_PCI0" diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 36a15e00ca..956f11cb88 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -467,7 +467,7 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) link_sink(c, unpack(sinks)) end -function reconfigurable_multi(f, graph, conf, ...) +function reconfigurable(f, graph, conf, ...) local args = {...} local function setup_fn(conf) local graph = config.new() @@ -499,18 +499,3 @@ function reconfigurable_multi(f, graph, conf, ...) follower_pids = { follower_pid }, schema_name = 'snabb-softwire-v1'}) end - -function reconfigurable_uni(f, graph, conf, ...) - local args = {...} - local function setup_fn(conf) - local graph = config.new() - f(graph, conf, unpack(args)) - return graph - end - - config.app(graph, 'leader', leader.Leader, - { setup_fn = setup_fn, initial_configuration = conf, - follower_pids = { S.getpid() }, - schema_name = 'snabb-softwire-v1'}) - config.app(graph, "follower", follower.Follower, {}) -end From 3e3281998c7c1f3636b1b379be890bc1d3aeb011 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Wed, 7 Dec 2016 13:02:30 +0100 Subject: [PATCH 395/631] Set up 'group' in the leader/parent process --- src/core/worker.lua | 1 + src/program/lwaftr/selftest.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index 41f73ae287..3dbc519ced 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -24,6 +24,7 @@ end -- Start a named worker to execute the given Lua code (a string). function start (name, luacode) + shm.mkdir(shm.resolve("group")) local pid = S.fork() if pid == 0 then -- First we perform some initialization functions and then we diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh index 7133693df3..108384f542 100755 --- a/src/program/lwaftr/selftest.sh +++ b/src/program/lwaftr/selftest.sh @@ -37,6 +37,7 @@ echo "Testing snabb lwaftr run" sudo ./snabb lwaftr run -D 0.1 --conf ${TDIR}/icmp_on_fail.conf \ --on-a-stick "$SNABB_PCI0" +# This needs to be 1 second, not 0.1 second, or it can mask DMA/setup problems echo "Testing snabb lwaftr run --reconfigurable" -sudo ./snabb lwaftr run -D 0.1 --reconfigurable \ +sudo ./snabb lwaftr run -D 1 --reconfigurable \ --conf ${TDIR}/icmp_on_fail.conf --on-a-stick "$SNABB_PCI0" From 3b3b373812cf3c7e07a8087b7e1f78060632b3e1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 13:26:23 +0100 Subject: [PATCH 396/631] Re-enable lwaftr CSV stats After the multiprocess merge, this facility needed some adaptation to keep on working. --- src/program/lwaftr/bench/bench.lua | 13 +++++++--- src/program/lwaftr/csv_stats.lua | 41 ++++++++++++++++++++---------- src/program/lwaftr/run/run.lua | 20 ++++++++------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index b8f025d67d..b1cb5a606d 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -63,13 +63,20 @@ function run(args) end app.configure(graph) - local function start_sampling() - local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) + local function start_sampling_for_pid(pid) + local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra, pid) csv:add_app('sinkv4', { 'input' }, { input=opts.hydra and 'decap' or 'Decap.' }) csv:add_app('sinkv6', { 'input' }, { input=opts.hydra and 'encap' or 'Encap.' }) csv:activate() end - --timer.activate(timer.new('spawn_csv_stats', start_sampling, 1e6)) + if opts.reconfigurable then + for _,pid in ipairs(app.configuration.apps['leader'].arg.follower_pids) do + local function start_sampling() start_sampling_for_pid(pid) end + timer.activate(timer.new('spawn_csv_stats', start_sampling, 100e6)) + end + else + start_sampling_for_pid(S.getpid()) + end app.busywait = true app.main({duration=opts.duration}) diff --git a/src/program/lwaftr/csv_stats.lua b/src/program/lwaftr/csv_stats.lua index 962e128918..3852876c32 100644 --- a/src/program/lwaftr/csv_stats.lua +++ b/src/program/lwaftr/csv_stats.lua @@ -1,11 +1,27 @@ module(..., package.seeall) +local S = require("syscall") +local shm = require("core.shm") local timer = require("core.timer") local engine = require("core.app") +local config = require("core.config") local counter = require("core.counter") CSVStatsTimer = {} +local function open_link_counters(pid) + local counters = {} + for _, linkspec in ipairs(shm.children("/"..pid.."/links")) do + local fa, fl, ta, tl = config.parse_link(linkspec) + local link = shm.open_frame("/"..pid.."/links/"..linkspec) + if not counters[fa] then counters[fa] = {input={},output={}} end + if not counters[ta] then counters[ta] = {input={},output={}} end + counters[fa].output[fl] = link + counters[ta].input[tl] = link + end + return counters +end + -- A timer that monitors packet rate and bit rate on a set of links, -- printing the data out to a CSV file. -- @@ -27,10 +43,12 @@ CSVStatsTimer = {} -- encap_mpps,2,3.407569,mpps -- encap_gbps,2,16.083724,gbps -- -function CSVStatsTimer:new(filename, hydra_mode) +function CSVStatsTimer:new(filename, hydra_mode, pid) local file = filename and io.open(filename, "w") or io.stdout local o = { hydra_mode=hydra_mode, link_data={}, file=file, period=1, header = hydra_mode and "benchmark,id,score,unit" or "Time (s)" } + o.pid = pid or S.getpid() + o.links_by_app = open_link_counters(pid) return setmetatable(o, {__index = CSVStatsTimer}) end @@ -40,29 +58,24 @@ end -- human-readable names, for the column headers. function CSVStatsTimer:add_app(id, links, link_names) local function add_link_data(name, link) - local link_name = link_names and link_names[name] or name + local link_name = link_names[name] or name if not self.hydra_mode then local h = (',%s MPPS,%s Gbps'):format(link_name, link_name) self.header = self.header..h end local data = { link_name = link_name, - txpackets = link.stats.txpackets, - txbytes = link.stats.txbytes, + txpackets = link.txpackets, + txbytes = link.txbytes, } table.insert(self.link_data, data) end - local app = assert(engine.app_table[id], "App named "..id.." not found") - if links then - for _,name in ipairs(links) do - local link = app.input[name] or app.output[name] - assert(link, "Link named "..name.." not found in "..id) - add_link_data(name, link) - end - else - for name,link in pairs(app.input) do add_link_data(name, link) end - for name,link in pairs(app.output) do add_link_data(name, link) end + local app = assert(self.links_by_app[id], "App named "..id.." not found") + for _,name in ipairs(links) do + local link = app.input[name] or app.output[name] + assert(link, "Link named "..name.." not found in "..id) + add_link_data(name, link) end end diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index c952a242f2..c3301286d6 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -166,15 +166,9 @@ function run(args) timer.activate(t) end - -- In reconfigurable mode, the app graph only gets populated later, - -- so we have to defer our timer creation. - local function later(f, when) - timer.activate(timer.new("later", f, when or 30e6)) - end - if opts.verbosity >= 1 then - function add_csv_stats() - local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra) + function add_csv_stats_for_pid(pid) + local csv = csv_stats.CSVStatsTimer:new(opts.bench_file, opts.hydra, pid) -- Link names like "tx" are from the app's perspective, but -- these labels are from the perspective of the lwAFTR as a -- whole so they are reversed. @@ -191,7 +185,15 @@ function run(args) end csv:activate() end - later(add_csv_stats) + if opts.reconfigurable then + local pids = engine.configuration.apps['leader'].arg.follower_pids + for _,pid in ipairs(pids) do + local function start_sampling() add_csv_stats_for_pid(pid) end + timer.activate(timer.new('spawn_csv_stats', start_sampling, 100e6)) + end + else + add_csv_stats_for_pid(S.getpid()) + end end if opts.ingress_drop_monitor then From 9bea7fecca2d29fdfd7a91a1823d65986ff1820f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 13:57:39 +0000 Subject: [PATCH 397/631] Unbreak ingress drop monitor. --- src/program/lwaftr/run/run.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 094a74bd06..d92cd20853 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -206,11 +206,13 @@ function run(args) end if opts.ingress_drop_monitor then - function add_ingress_drop_monitor() + if opts.reconfigurable then + io.stderr:write("Warning: Ingress drop monitor not yet supported ".. + "in multiprocess mode.\n") + else local mon = ingress_drop_monitor.new({action=opts.ingress_drop_monitor}) timer.activate(mon:timer()) end - later(add_ingress_drop_monitor) end engine.busywait = true From 2d5c76b356e956d78172d567ac36221178bc8bbf Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 14:42:28 +0000 Subject: [PATCH 398/631] Allow for asynchrony between main and worker lwaftr for stats --- src/program/lwaftr/bench/bench.lua | 13 +++++++++++-- src/program/lwaftr/run/run.lua | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 5d301ba06b..080dc475ee 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -71,8 +71,17 @@ function run(args) end if opts.reconfigurable then for _,pid in ipairs(app.configuration.apps['leader'].arg.follower_pids) do - local function start_sampling() start_sampling_for_pid(pid) end - timer.activate(timer.new('spawn_csv_stats', start_sampling, 100e6)) + -- The worker will be fed its configuration by the + -- leader, but we don't know when that will all be ready. + -- Just retry if this doesn't succeed. + local function start_sampling() + if not pcall(start_sampling_for_pid, pid) then + io.stderr:write("Waiting on follower "..pid.." to start ".. + "before recording statistics...\n") + timer.activate(timer.new('retry_csv', start_sampling, 1e9)) + end + end + timer.activate(timer.new('spawn_csv_stats', start_sampling, 10e6)) end else start_sampling_for_pid(S.getpid()) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index d92cd20853..6d74f63755 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -197,8 +197,17 @@ function run(args) if opts.reconfigurable then local pids = engine.configuration.apps['leader'].arg.follower_pids for _,pid in ipairs(pids) do - local function start_sampling() add_csv_stats_for_pid(pid) end - timer.activate(timer.new('spawn_csv_stats', start_sampling, 100e6)) + local function start_sampling() + -- The worker will be fed its configuration by the + -- leader, but we don't know when that will all be ready. + -- Just retry if this doesn't succeed. + if not pcall(add_csv_stats_for_pid, pid) then + io.stderr:write("Waiting on follower "..pid.." to start ".. + "before recording statistics...\n") + timer.activate(timer.new('retry_csv', start_sampling, 2e9)) + end + end + timer.activate(timer.new('spawn_csv_stats', start_sampling, 50e6)) end else add_csv_stats_for_pid(S.getpid()) From 5c2bebc540bbd3477dd217fcd6243db7e79b9a71 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 7 Dec 2016 15:46:09 +0100 Subject: [PATCH 399/631] Fix -h and --help for the ps program There was an issue with the call to dogetopt where the help option expected an argument, of course, this isn't the case. --- src/program/ps/ps.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index 048c21259f..eede9f3f8e 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -18,7 +18,7 @@ local function parse_args (args) local preferpid = false function opt.h (arg) usage(0) end function opt.p (arg) preferpid = true end - args = lib.dogetopt(args, opt, "h:p", {help='h', pid='p'}) + args = lib.dogetopt(args, opt, "hp", {help='h', pid='p'}) if #args ~= 0 then usage(1) end return preferpid end From 57aa527cb2b49c5e070153a82b8d53f8892480ce Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 7 Dec 2016 16:12:16 +0100 Subject: [PATCH 400/631] Snabb lwAFTR v3.1.2 change log --- src/program/lwaftr/doc/CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 999002f012..c9423cceb8 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,19 @@ # Change Log +## [3.1.2] - 2016-12-07 + + * Re-enabled multi-process mode for --reconfigurable "snabb lwaftr + run", including support for "snabb config get-state". + + * Improve memory consumption when parsing big configurations, such as a + binding table with a million entries. + + * Re-enable CSV-format statistics for "snabb lwaftr bench" and "snabb + lwaftr run", which were disabled while we landed multiprocess + support. + + * Fix "snabb ps --help". + ## [3.1.1] - 2016-12-06 A hotfix to work around bugs in multiprocess support when using Intel From 9efc299746c3c2503fff4097ecaedee1d8139c9e Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 7 Dec 2016 11:47:27 +0000 Subject: [PATCH 401/631] Redo by-name query support in 'lwaftr query' --- src/program/lwaftr/query/README | 18 ++++----- src/program/lwaftr/query/query.lua | 65 +++++++++++++++++------------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/src/program/lwaftr/query/README b/src/program/lwaftr/query/README index c22d9e2623..e186e82eb6 100644 --- a/src/program/lwaftr/query/README +++ b/src/program/lwaftr/query/README @@ -1,20 +1,18 @@ Usage: - query [OPTIONS] [] [] + query [OPTIONS] [] [] Options: - -h, --help Print usage information. - -l, --list-all List all available counter names. + -n, --name Name of snabb process. + -h, --help Print usage information. + -l, --list-all List all available counter names. Display current statistics from lwAFTR counters for a running Snabb instance -with . can be either a PID or a string ID: +with . - * If it is a PID, should exists at /var/run/snabb/. - * If it is a string ID, should match the value defined - in /var/run/snabb/*/nic/id. - -If is not supplied and there is only one Snabb instance, "query" will -connect to that instance. +If is empty and there is only one Snabb instance, "query" will +connect to that instance. It is possible to query a process by name, passing +the flag --name. If is set, only counters partially matching are listed. diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 00cbb7242f..1aa3f9902a 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -1,16 +1,16 @@ module(..., package.seeall) +local S = require("syscall") local counter = require("core.counter") -local ffi = require("ffi") local lib = require("core.lib") local lwcounter = require("apps.lwaftr.lwcounter") -local lwtypes = require("apps.lwaftr.lwtypes") local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") local top = require("program.top.top") local select_snabb_instance = top.select_snabb_instance -local keys = lwutil.keys +local keys, fatal = lwutil.keys, lwutil.fatal +local named_program_root = engine.named_program_root -- Get the counter dir from the code. local counters_dir = lwcounter.counters_dir @@ -29,22 +29,20 @@ local function is_counter_name (name) return lwcounter.counter_names[name] ~= nil end -local function pidof(maybe_pid) - if tonumber(maybe_pid) then return maybe_pid end - local name_id = maybe_pid - for _, pid in ipairs(shm.children("/")) do - local path = "/"..pid.."/nic/id" - if shm.exists(path) then - local lwaftr_id = shm.open(path, lwtypes.lwaftr_id_type) - if ffi.string(lwaftr_id.value) == name_id then - return pid - end +local function get_pid_by_name (name) + for _, program in pairs(shm.children("/by-name")) do + if name == program then + local fq = named_program_root .. "/" .. program + local piddir = S.readlink(fq) + return lib.basename(piddir) end end end function parse_args (raw_args) local handlers = {} + local opts = {} + local name function handlers.h() show_usage(0) end function handlers.l () for _, name in ipairs(sort(lwcounter.counter_names)) do @@ -52,25 +50,36 @@ function parse_args (raw_args) end main.exit(0) end - local args = lib.dogetopt(raw_args, handlers, "hl", - { help="h", ["list-all"]="l" }) - if #args > 2 then show_usage(1) end - if #args == 2 then - return args[1], args[2] + function handlers.n (arg) + opts.name = assert(arg) end - if #args == 1 then - local arg = args[1] - if is_counter_name(arg) then - return nil, arg - else - local pid = pidof(arg) - if not pid then - error(("Couldn't find PID for argument '%s'"):format(arg)) + local args = lib.dogetopt(raw_args, handlers, "hln:", + { help="h", ["list-all"]="l", name="n" }) + if opts.name then + if #args > 1 then + print("Too many arguments.") + show_usage(1) + end + local pid = get_pid_by_name(opts.name) + if not pid then + fatal(("Couldn't find process with name '%s'"):format(opts.name)) + end + local counter_name = args and args[1] + return pid, counter_name + else + if #args == 2 then + return unpack(args) + elseif #args == 1 then + if is_counter_name(args[1]) then + return nil, args[1] + else + return args[1] end - return pid, nil + elseif #args > 2 then + print("Too many arguments.") + show_usage(1) end end - return nil, nil end local function read_counters (tree) From eb5088f48109625d28b93f3383ee25e381f9eab5 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 7 Dec 2016 12:05:47 +0000 Subject: [PATCH 402/631] Redo by-name query support in 'snabbvmx query' --- src/program/snabbvmx/lwaftr/lwaftr.lua | 5 +---- src/program/snabbvmx/query/query.lua | 13 ++++++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/program/snabbvmx/lwaftr/lwaftr.lua b/src/program/snabbvmx/lwaftr/lwaftr.lua index 61ed8558aa..37e9c5c09b 100644 --- a/src/program/snabbvmx/lwaftr/lwaftr.lua +++ b/src/program/snabbvmx/lwaftr/lwaftr.lua @@ -138,10 +138,7 @@ function run(args) intel10g.ring_buffer_size(ring_buffer_size) - if id then - local lwaftr_id = shm.create("nic/id", lwtypes.lwaftr_id_type) - lwaftr_id.value = id - end + if id then engine.claim_name(id) end local vlan = false local mtu = DEFAULT_MTU diff --git a/src/program/snabbvmx/query/query.lua b/src/program/snabbvmx/query/query.lua index 1a3fd4debd..02e3ac73c0 100644 --- a/src/program/snabbvmx/query/query.lua +++ b/src/program/snabbvmx/query/query.lua @@ -98,15 +98,22 @@ local function print_counters (pid, dir) print((" "):format(dir)) end +local named_program_root = engine.named_program_root + +function get_name_by_pid (pid) + local fq = named_program_root .. "/" .. pid + local namedir = S.readlink(fq) + return lib.basename(namedir) +end + function run (raw_args) parse_args(raw_args) print("") local pids = {} local pids_name = {} for _, pid in ipairs(shm.children("/")) do - if shm.exists("/"..pid.."/nic/id") then - local lwaftr_id = shm.open("/"..pid.."/nic/id", lwtypes.lwaftr_id_type) - local instance_id_name = ffi.string(lwaftr_id.value) + if shm.exists("/"..pid.."/name") then + local instance_id_name = get_name_by_pid(pid) local instance_id = instance_id_name and instance_id_name:match("(%d+)") if instance_id then pids[instance_id] = pid From ee95c09e6251fe6b3d735b0570927451dbf03fa3 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 7 Dec 2016 12:06:20 +0000 Subject: [PATCH 403/631] Redo by-name query support in 'snabbvmx top' --- src/apps/lwaftr/lwutil.lua | 11 +++++++ src/program/lwaftr/query/query.lua | 12 +------- src/program/snabbvmx/top/README | 9 ++++-- src/program/snabbvmx/top/top.lua | 46 +++++++++++------------------- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/apps/lwaftr/lwutil.lua b/src/apps/lwaftr/lwutil.lua index 86cf56a6d2..4084e75e3f 100644 --- a/src/apps/lwaftr/lwutil.lua +++ b/src/apps/lwaftr/lwutil.lua @@ -6,6 +6,7 @@ local S = require("syscall") local bit = require("bit") local ffi = require("ffi") local lib = require("core.lib") +local shm = require("core.shm") local band = bit.band local cast = ffi.cast @@ -116,3 +117,13 @@ function nic_exists(pci_addr) return dir_exists(("%s/%s"):format(devices, pci_addr)) or dir_exists(("%s/0000:%s"):format(devices, pci_addr)) end + +function get_pid_by_name (name) + for _, program in pairs(shm.children("/by-name")) do + if name == program then + local fq = engine.named_program_root .. "/" .. program + local piddir = S.readlink(fq) + return lib.basename(piddir) + end + end +end diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 1aa3f9902a..d1c3bb23ed 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -10,7 +10,7 @@ local top = require("program.top.top") local select_snabb_instance = top.select_snabb_instance local keys, fatal = lwutil.keys, lwutil.fatal -local named_program_root = engine.named_program_root +local get_pid_by_name = lwutil.get_pid_by_name -- Get the counter dir from the code. local counters_dir = lwcounter.counters_dir @@ -29,16 +29,6 @@ local function is_counter_name (name) return lwcounter.counter_names[name] ~= nil end -local function get_pid_by_name (name) - for _, program in pairs(shm.children("/by-name")) do - if name == program then - local fq = named_program_root .. "/" .. program - local piddir = S.readlink(fq) - return lib.basename(piddir) - end - end -end - function parse_args (raw_args) local handlers = {} local opts = {} diff --git a/src/program/snabbvmx/top/README b/src/program/snabbvmx/top/README index 9e6f286066..a784a6a6af 100644 --- a/src/program/snabbvmx/top/README +++ b/src/program/snabbvmx/top/README @@ -1,13 +1,16 @@ Usage: top [OPTIONS] [] - -h, --help - Print usage information. +Options: + -n, --name Query by process name. + -h, --help Print usage information. -Display realtime lwaftr statistics for a running lwaftr Snabb Switch +Display realtime lwaftr statistics for a running Snabb's lwaftr instance with . If is not supplied and there is only one Snabb Switch instance, top will connect to that instance. +It is possible to query a process by name, passing the flag --name. + Example output: diff --git a/src/program/snabbvmx/top/top.lua b/src/program/snabbvmx/top/top.lua index b337fdfb60..6a176508f4 100644 --- a/src/program/snabbvmx/top/top.lua +++ b/src/program/snabbvmx/top/top.lua @@ -10,37 +10,15 @@ local shm = require("core.shm") local top = require("program.top.top") local C = ffi.C -local fatal = lwutil.fatal +local fatal, get_pid_by_name = lwutil.fatal, lwutil.get_pid_by_name local long_opts = { - help = "h" + help = "h", + name = "n" } local function clearterm () io.write('\027[2J') end -local function select_snabb_instance_by_id (target_id) - local pids = shm.children("/") - for _, pid in ipairs(pids) do - local path = "/"..pid.."/nic/id" - if shm.exists(path) then - local lwaftr_id = shm.open(path, lwtypes.lwaftr_id_type) - if ffi.string(lwaftr_id.value) == target_id then - return pid - end - end - end - print(("Couldn't find instance with id '%s'"):format(target_id)) - main.exit(1) -end - -local function select_snabb_instance (id) - if not id or tonumber(id) then - return top.select_snabb_instance(id) - else - return select_snabb_instance_by_id(id) - end -end - local counter_names = (function () local counters = { "in-%s-packets", -- rcvdPacket @@ -207,17 +185,27 @@ end local function parse_args (args) local handlers = {} + local opts = {} function handlers.h () show_usage(0) end - args = lib.dogetopt(args, handlers, "h", long_opts) + function handlers.n (arg) + opts.name = assert(arg) + end + args = lib.dogetopt(args, handlers, "hn:", long_opts) if #args > 1 then show_usage(1) end - return args[1] + return opts, args[1] end function run (args) - local target_pid = parse_args(args) - local instance_tree = "/"..select_snabb_instance(target_pid) + local opts, target_pid = parse_args(args) + if opts.name then + target_pid = get_pid_by_name(opts.name) + if not target_pid then + fatal(("Couldn't find process with name '%s'"):format(opts.name)) + end + end + local instance_tree = "/" .. top.select_snabb_instance(target_pid) if not has_lwaftr_app(instance_tree) then fatal("Selected instance doesn't include lwaftr app") end From 97c52b1c43b0622f4686f44a4ff38b4e97d185d6 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 7 Dec 2016 15:02:27 +0000 Subject: [PATCH 404/631] Redo by-name query support in 'lwaftr monitor' --- src/program/lwaftr/monitor/README | 21 +++--- src/program/lwaftr/monitor/monitor.lua | 91 ++++++++------------------ 2 files changed, 39 insertions(+), 73 deletions(-) diff --git a/src/program/lwaftr/monitor/README b/src/program/lwaftr/monitor/README index 400de79b16..0900511286 100644 --- a/src/program/lwaftr/monitor/README +++ b/src/program/lwaftr/monitor/README @@ -1,22 +1,27 @@ Usage: - monitor ACTION|IPV4_ADDRESS [PID] + monitor IPV4_ADDRESS [PID] - -h, --help - Print usage information. +Options: + + -n, --name Name of snabb process. + -h, --help Print usage information. Arguments: - ACTION Action to perform: 'none' or 'all' - IPV4_ADDRESS IPv4 address to mirror - PID PID value of Snabb process. + IPV4_ADDRESS IPv4 address to mirror. + PID PID value of Snabb process. Sets the value of 'v4v6_mirror' counter to IPV4_ADDRESS. The 'v4v6_mirror' counter is defined for all lwAFTR instances running in mirroring mode. Matching packets will be mirrored to the tap interface set by the original lwAFTR process. -When action is 'none' IPV4_ADDRESS is set to '0.0.0.0'. When ACTION is 'all' -IPV4_ADDRESS is set to '255.255.255.255'. +It is possible to query a process by name, passing the flag --name. + +IPV4_ADDRESS can take two special values: + +- 'none', which is equivalent to '0.0.0.0'. +- 'all', which is equivalent to '255.255.255.255'. PID value is used to retrieve the lwAFTR instance. If PID is not set, the most recent active lwAFTR instance for which 'v4v6_mirror' is defined is used. diff --git a/src/program/lwaftr/monitor/monitor.lua b/src/program/lwaftr/monitor/monitor.lua index 067179d394..d1927c1e26 100644 --- a/src/program/lwaftr/monitor/monitor.lua +++ b/src/program/lwaftr/monitor/monitor.lua @@ -6,11 +6,14 @@ local lib = require("core.lib") local lwtypes = require("apps.lwaftr.lwtypes") local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") +local top = require("program.top.top") -local fatal = lwutil.fatal +local fatal, get_pid_by_name = lwutil.fatal, lwutil.get_pid_by_name +local select_snabb_instance = top.select_snabb_instance local long_opts = { - help = "h" + help = "h", + name = "n", } local MIRROR_NOTHING = "0.0.0.0" @@ -23,64 +26,24 @@ end local function parse_args (args) local handlers = {} + local opts = {} function handlers.h () usage(0) end - args = lib.dogetopt(args, handlers, "h", long_opts) - if #args < 1 or #args > 2 then usage(1) end - - -- Return address and pid. - if #args == 1 then - local maybe_pid = tonumber(args[1]) - if maybe_pid then - return MIRROR_NOTHING, maybe_pid - end - return args[1] - end - return args[1], args[2] -end - -local function find_pid_by_id (id) - for _, pid in ipairs(shm.children("/")) do - local path = "/"..pid.."/nic/id" - if shm.exists(path) then - local lwaftr_id = shm.open(path, lwtypes.lwaftr_id_type) - if ffi.string(lwaftr_id.value) == id then - return pid - end - end + function handlers.n (arg) + opts.name = assert(arg) end + args = lib.dogetopt(args, handlers, "hn:", long_opts) + if #args < 1 or #args > 2 then usage(1) end + return opts, unpack(args) end local function find_mirror_path (pid) - -- Check process has v4v6_mirror defined. - if pid then - -- Pid is an id. - if not tonumber(pid) then - pid = find_pid_by_id(pid) - if not pid then - fatal("Invalid lwAFTR id '%s'"):format(pid) - end - end - -- Pid exists? - if not shm.exists("/"..pid) then - fatal(("No Snabb instance with pid '%d'"):format(pid)) - end - -- Check process has v4v6_mirror defined. - local path = "/"..pid.."/v4v6_mirror" - if not shm.exists(path) then - fatal(("lwAFTR process '%d' is not running in mirroring mode"):format(pid)) - end - return path, pid - end - - -- Return first process which has v4v6_mirror defined. - for _, pid in ipairs(shm.children("/")) do - local path = "/"..pid.."/v4v6_mirror" - if shm.exists(path) then - return path, pid - end + local path = "/"..pid.."/v4v6_mirror" + if not shm.exists(path) then + fatal(("lwAFTR process '%d' is not running in mirroring mode"):format(pid)) end + return path, pid end local function set_mirror_address (address, path) @@ -91,10 +54,8 @@ local function set_mirror_address (address, path) -- Validate address. if address == "none" then - print("Monitor none") address = MIRROR_NOTHING elseif address == "all" then - print("Monitor all") address = MIRROR_EVERYTHING else if not ipv4:pton(address) then @@ -107,19 +68,19 @@ local function set_mirror_address (address, path) local v4v6_mirror = shm.open(path, "struct { uint32_t ipv4; }") v4v6_mirror.ipv4 = ipv4_num shm.unmap(v4v6_mirror) + + return address end function run (args) - local address, pid = parse_args(args) - local path, pid_number = find_mirror_path(pid) - if not path then - fatal("Couldn't find lwAFTR process running in mirroring mode") - end - - set_mirror_address(address, path) - io.write(("Mirror address set to '%s'"):format(address)) - if not tonumber(pid) then - io.write((" in PID '%d'"):format(pid_number)) + local opts, address, pid = parse_args(args) + if opts.name then + pid = get_pid_by_name(opts.name) + if not pid then + fatal(("Couldn't find process with name '%s'"):format(opts.name)) + end end - io.write("\n") + local path, pid = find_mirror_path(top.select_snabb_instance(pid)) + address = set_mirror_address(address, path) + print(("Mirror address set to '%s' in PID '%s'"):format(address, pid)) end From 2ae3ed3715ce88801a121869c1421853bc4da60e Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 7 Dec 2016 18:10:13 +0000 Subject: [PATCH 405/631] Cast pid argument to a string List of instances are pid strings. The pid passed as an argument could be either a string or a number. Force cast to string to guarantee values can be successfully compared. --- src/program/top/top.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/program/top/top.lua b/src/program/top/top.lua index 01f3abd1fa..c9754dfbda 100644 --- a/src/program/top/top.lua +++ b/src/program/top/top.lua @@ -47,6 +47,7 @@ function select_snabb_instance (pid) local instances = compute_snabb_instances() if pid then + pid = tostring(pid) -- Try to use given pid for _, instance in ipairs(instances) do if instance == pid then return pid end From b9eee442bc560e3de0bdfd4b1e70696d8abe22b3 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 7 Dec 2016 18:12:25 +0000 Subject: [PATCH 406/631] Replace functions 'get_pid_by_name' and 'get_name_by_pid' --- src/apps/lwaftr/lwutil.lua | 11 ------- src/program/lwaftr/monitor/monitor.lua | 5 +-- src/program/lwaftr/query/query.lua | 43 +++++++++----------------- src/program/snabbvmx/query/query.lua | 15 +++++---- src/program/snabbvmx/top/top.lua | 5 +-- 5 files changed, 27 insertions(+), 52 deletions(-) diff --git a/src/apps/lwaftr/lwutil.lua b/src/apps/lwaftr/lwutil.lua index 4084e75e3f..86cf56a6d2 100644 --- a/src/apps/lwaftr/lwutil.lua +++ b/src/apps/lwaftr/lwutil.lua @@ -6,7 +6,6 @@ local S = require("syscall") local bit = require("bit") local ffi = require("ffi") local lib = require("core.lib") -local shm = require("core.shm") local band = bit.band local cast = ffi.cast @@ -117,13 +116,3 @@ function nic_exists(pci_addr) return dir_exists(("%s/%s"):format(devices, pci_addr)) or dir_exists(("%s/0000:%s"):format(devices, pci_addr)) end - -function get_pid_by_name (name) - for _, program in pairs(shm.children("/by-name")) do - if name == program then - local fq = engine.named_program_root .. "/" .. program - local piddir = S.readlink(fq) - return lib.basename(piddir) - end - end -end diff --git a/src/program/lwaftr/monitor/monitor.lua b/src/program/lwaftr/monitor/monitor.lua index d1927c1e26..810c1e7fa5 100644 --- a/src/program/lwaftr/monitor/monitor.lua +++ b/src/program/lwaftr/monitor/monitor.lua @@ -8,7 +8,7 @@ local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") local top = require("program.top.top") -local fatal, get_pid_by_name = lwutil.fatal, lwutil.get_pid_by_name +local fatal = lwutil.fatal local select_snabb_instance = top.select_snabb_instance local long_opts = { @@ -75,7 +75,8 @@ end function run (args) local opts, address, pid = parse_args(args) if opts.name then - pid = get_pid_by_name(opts.name) + local programs = engine.enumerate_named_programs(opts.name) + pid = programs[opts.name] if not pid then fatal(("Couldn't find process with name '%s'"):format(opts.name)) end diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index d1c3bb23ed..929cf06e6e 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -8,9 +8,7 @@ local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") local top = require("program.top.top") -local select_snabb_instance = top.select_snabb_instance local keys, fatal = lwutil.keys, lwutil.fatal -local get_pid_by_name = lwutil.get_pid_by_name -- Get the counter dir from the code. local counters_dir = lwcounter.counters_dir @@ -45,31 +43,8 @@ function parse_args (raw_args) end local args = lib.dogetopt(raw_args, handlers, "hln:", { help="h", ["list-all"]="l", name="n" }) - if opts.name then - if #args > 1 then - print("Too many arguments.") - show_usage(1) - end - local pid = get_pid_by_name(opts.name) - if not pid then - fatal(("Couldn't find process with name '%s'"):format(opts.name)) - end - local counter_name = args and args[1] - return pid, counter_name - else - if #args == 2 then - return unpack(args) - elseif #args == 1 then - if is_counter_name(args[1]) then - return nil, args[1] - else - return args[1] - end - elseif #args > 2 then - print("Too many arguments.") - show_usage(1) - end - end + if #args > 2 then show_usage(1) end + return opts, unpack(args) end local function read_counters (tree) @@ -113,7 +88,17 @@ local function print_counters (tree, filter) end function run (raw_args) - local target_pid, counter_name = parse_args(raw_args) - local instance_tree = select_snabb_instance(target_pid) + local opts, pid, counter_name = parse_args(raw_args) + if tostring(pid) and not counter_name then + counter_name, pid = pid, nil + end + if opts.name then + local programs = engine.enumerate_named_programs(opts.name) + pid = programs[opts.name] + if not pid then + fatal(("Couldn't find process with name '%s'"):format(opts.name)) + end + end + local instance_tree = top.select_snabb_instance(pid) print_counters(instance_tree, counter_name) end diff --git a/src/program/snabbvmx/query/query.lua b/src/program/snabbvmx/query/query.lua index 02e3ac73c0..f6bd1d8523 100644 --- a/src/program/snabbvmx/query/query.lua +++ b/src/program/snabbvmx/query/query.lua @@ -31,7 +31,6 @@ local function parse_args (raw_args) local args = lib.dogetopt(raw_args, handlers, "h", { help="h" }) if #args > 0 then show_usage(1) end - return nil end local function read_counters (tree, app_name) @@ -98,12 +97,10 @@ local function print_counters (pid, dir) print((" "):format(dir)) end -local named_program_root = engine.named_program_root - -function get_name_by_pid (pid) - local fq = named_program_root .. "/" .. pid - local namedir = S.readlink(fq) - return lib.basename(namedir) +local function transpose (t) + local ret = {} + for k, v in pairs(t) do ret[v] = k end + return ret end function run (raw_args) @@ -111,9 +108,11 @@ function run (raw_args) print("") local pids = {} local pids_name = {} + local named_programs = transpose(engine.enumerate_named_programs()) + for _, pid in ipairs(shm.children("/")) do if shm.exists("/"..pid.."/name") then - local instance_id_name = get_name_by_pid(pid) + local instance_id_name = named_programs[tonumber(pid)] local instance_id = instance_id_name and instance_id_name:match("(%d+)") if instance_id then pids[instance_id] = pid diff --git a/src/program/snabbvmx/top/top.lua b/src/program/snabbvmx/top/top.lua index 6a176508f4..aa872dfa91 100644 --- a/src/program/snabbvmx/top/top.lua +++ b/src/program/snabbvmx/top/top.lua @@ -10,7 +10,7 @@ local shm = require("core.shm") local top = require("program.top.top") local C = ffi.C -local fatal, get_pid_by_name = lwutil.fatal, lwutil.get_pid_by_name +local fatal = lwutil.fatal local long_opts = { help = "h", @@ -200,7 +200,8 @@ end function run (args) local opts, target_pid = parse_args(args) if opts.name then - target_pid = get_pid_by_name(opts.name) + local programs = engine.enumerate_named_programs(opts.name) + target_pid = programs[opts.name] if not target_pid then fatal(("Couldn't find process with name '%s'"):format(opts.name)) end From 785221bc4d6fedff1ce7ce026f734a87c0829bcc Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Thu, 8 Dec 2016 09:21:01 +0100 Subject: [PATCH 407/631] Yang config doc fixes (#643) Improvements to the YANG config docs, plus aligning the README/README.inc structure of src/program/config subcommands with other existing commands. --- src/program/config/README | 15 ++++++++++++ src/program/config/README.inc | 4 +--- src/program/config/README.md | 13 +++++++++-- src/program/config/data_format/README | 27 ++++++++++++++++++++++ src/program/config/data_format/README.inc | 28 +---------------------- src/program/config/get_state/README | 21 ++++++++++++++++- src/program/config/get_state/README.inc | 14 +----------- src/program/config/load/README | 15 ++++++++++++ src/program/config/load/README.inc | 16 +------------ 9 files changed, 92 insertions(+), 61 deletions(-) create mode 100644 src/program/config/README mode change 100644 => 120000 src/program/config/README.inc create mode 100644 src/program/config/data_format/README mode change 100644 => 120000 src/program/config/data_format/README.inc mode change 120000 => 100644 src/program/config/get_state/README mode change 100644 => 120000 src/program/config/get_state/README.inc create mode 100644 src/program/config/load/README mode change 100644 => 120000 src/program/config/load/README.inc diff --git a/src/program/config/README b/src/program/config/README new file mode 100644 index 0000000000..8f5a4ca6ee --- /dev/null +++ b/src/program/config/README @@ -0,0 +1,15 @@ +Usage: + snabb config add + snabb config get + snabb config get-state + snabb config listen + snabb config load + snabb config remove + snabb config set + +Use --help for per-command usage. +Example: + snabb config add --help + +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. diff --git a/src/program/config/README.inc b/src/program/config/README.inc deleted file mode 100644 index f49754795e..0000000000 --- a/src/program/config/README.inc +++ /dev/null @@ -1,3 +0,0 @@ -Usage: - config derive-data-format [OPTIONS] [] - get-config SOCKET MSG diff --git a/src/program/config/README.inc b/src/program/config/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/README.md b/src/program/config/README.md index 668bf4f68d..eea6915fff 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -141,6 +141,14 @@ Likewise, to change the port for `1.2.3.4`, do: $ snabb config set ID /routes/route[addr=1.2.3.4]/port 7 ``` +If the element has a multiple-value key, you can use multiple XPath +selectors. For instance, if route elements had "addr port" as key, +you'd do: + +``` +$ snabb config get ID /routes/route[addr=1.2.3.4][port=1] +``` + The general rule for paths and value syntax is that if a name appears in the path, it won't appear in the value. Mostly this works as you would expect, but there are a couple of edge cases for instances of `list` and @@ -213,7 +221,8 @@ $ cat /tmp/my-configuration | snabb config set ID / ``` Resetting the whole configuration is such a common operation that it -has a special command that takes a file name instead of a path: +has a special command that takes a filesystem path instead of a schema +path: ``` $ snabb config load ID /tmp/my-configuration @@ -245,7 +254,7 @@ $ snabb config add ID /routes/route The `listen` interface supports all of these operations with a simple JSON protocol. `snabb config listen` reads JSON objects from `stdin`, parses them, relays their action to the data plane, and writes responses -out to `stdout`. The requests are be processed in order, but +out to `stdout`. The requests are processed in order, but asynchronously; `snabb config listen` doesn't wait for a response from the data plane before processing the next request. In this way, a NETCONF agent can pipeline a number of requests. diff --git a/src/program/config/data_format/README b/src/program/config/data_format/README new file mode 100644 index 0000000000..44f22c7567 --- /dev/null +++ b/src/program/config/data_format/README @@ -0,0 +1,27 @@ +Data Format Usage: + snabb config data-format + +This command produces an annotated yang data format from a schema file +which can be used to check a yang schema or help you write the data +configuration file. + +The output is an option or a container of options followed by the type +of value that is to be expected. In the case where there is a option +with a section in curly braces that will represent a nested structure +such as a list or a container. Comments preciding certain fields will +indicate if for example the block is a list, or comments may proceed +options specifying if they are for example mandatory. + +An exaaple of an option could be: + + port uint8; // mandatory between 0..11 + +This describes a configuration option "port" which takes an unsigned +integer with a value between 0 and 11. The option is required so must +have a value. An example for this field in the configuration could be: + + port 7; + +Example usage: + + $ snabb config data-format lib/yang/snabb-softwire.yang diff --git a/src/program/config/data_format/README.inc b/src/program/config/data_format/README.inc deleted file mode 100644 index 44f22c7567..0000000000 --- a/src/program/config/data_format/README.inc +++ /dev/null @@ -1,27 +0,0 @@ -Data Format Usage: - snabb config data-format - -This command produces an annotated yang data format from a schema file -which can be used to check a yang schema or help you write the data -configuration file. - -The output is an option or a container of options followed by the type -of value that is to be expected. In the case where there is a option -with a section in curly braces that will represent a nested structure -such as a list or a container. Comments preciding certain fields will -indicate if for example the block is a list, or comments may proceed -options specifying if they are for example mandatory. - -An exaaple of an option could be: - - port uint8; // mandatory between 0..11 - -This describes a configuration option "port" which takes an unsigned -integer with a value between 0 and 11. The option is required so must -have a value. An example for this field in the configuration could be: - - port 7; - -Example usage: - - $ snabb config data-format lib/yang/snabb-softwire.yang diff --git a/src/program/config/data_format/README.inc b/src/program/config/data_format/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/data_format/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/get_state/README b/src/program/config/get_state/README deleted file mode 120000 index d25b9bc34d..0000000000 --- a/src/program/config/get_state/README +++ /dev/null @@ -1 +0,0 @@ -README.inc \ No newline at end of file diff --git a/src/program/config/get_state/README b/src/program/config/get_state/README new file mode 100644 index 0000000000..6f2797e96b --- /dev/null +++ b/src/program/config/get_state/README @@ -0,0 +1,20 @@ +Usage: snabb config get-state [OPTION]... ID PATH +Get the state for a Snabb network function. + +Available options: + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + -h, --help Displays this message. + +Given an instance identifier and a schema path, display the current counter +values. + +If the --schema argument is not provided, "snabb config" will ask the data +plane for its native schema. The result will be printed on standard output. + +Typical usage: + +$ snabb config get-state lwaftr /softwire-state/ + +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. diff --git a/src/program/config/get_state/README.inc b/src/program/config/get_state/README.inc deleted file mode 100644 index cf0661304f..0000000000 --- a/src/program/config/get_state/README.inc +++ /dev/null @@ -1,13 +0,0 @@ -Usage: snabb config get-state [OPTION]... ID PATH -Get the state for a Snabb network function. - -Available options: - -s, --schema SCHEMA YANG data interface to request. - -r, --revision REVISION Require a specific revision of the YANG module. - -h, --help Displays this message. - -If the --schema argument is not provided, "snabb config" will ask the data -plane for its native schema. The result will be printed on standard output. - -See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md -for full documentation. diff --git a/src/program/config/get_state/README.inc b/src/program/config/get_state/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/get_state/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/load/README b/src/program/config/load/README new file mode 100644 index 0000000000..789078554f --- /dev/null +++ b/src/program/config/load/README @@ -0,0 +1,15 @@ +Usage: + snabb config load [OPTIONS] ID FILE + +Available options: + -s SCHEMA + --schema SCHEMA + --schema-name SCHEMA + + -r REVISION + --revision REVISION + --revision-date REVISION + + -h + --help + diff --git a/src/program/config/load/README.inc b/src/program/config/load/README.inc deleted file mode 100644 index 789078554f..0000000000 --- a/src/program/config/load/README.inc +++ /dev/null @@ -1,15 +0,0 @@ -Usage: - snabb config load [OPTIONS] ID FILE - -Available options: - -s SCHEMA - --schema SCHEMA - --schema-name SCHEMA - - -r REVISION - --revision REVISION - --revision-date REVISION - - -h - --help - diff --git a/src/program/config/load/README.inc b/src/program/config/load/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/load/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file From 15851a84d0407e9f5ffa0d0de6d357b57ba577c6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 8 Dec 2016 09:29:46 +0000 Subject: [PATCH 408/631] Schedule main and worker processes on different CPUS. This patch also allows ingress drop monitors and real-time mode to be enabled in the data plane. --- src/program/lwaftr/bench/bench.lua | 21 +++----- src/program/lwaftr/run/run.lua | 46 +++++++---------- src/program/lwaftr/setup.lua | 80 +++++++++++++++++++++++------- 3 files changed, 88 insertions(+), 59 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 080dc475ee..acc3c4bb9b 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -16,25 +16,17 @@ end function parse_args(args) local handlers = {} local opts = { bench_file = 'bench.csv' } + local scheduling = {} function handlers.D(arg) opts.duration = assert(tonumber(arg), "duration must be a number") assert(opts.duration >= 0, "duration can't be negative") end function handlers.cpu(arg) - cpu = tonumber(arg) + local cpu = tonumber(arg) if not cpu or cpu ~= math.floor(cpu) or cpu < 0 then fatal("Invalid cpu number: "..arg) end - - if opts.reconfigurable then - S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) - local wanted_node = numa.cpu_get_numa_node(cpu) - numa.bind_to_numa_node(wanted_node) - print("Bound to numa node:", wanted_node) - else - print("Bound to CPU:", cpu) - numa.bind_to_cpu(cpu) - end + scheduling.cpu = cpu end function handlers.n(arg) opts.name = assert(arg) end function handlers.b(arg) opts.bench_file = arg end @@ -45,20 +37,21 @@ function parse_args(args) help="h", hydra="y", ["bench-file"]="b", duration="D", name="n", cpu=1, reconfigurable = 0 }) if #args ~= 3 then show_usage(1) end - return opts, unpack(args) + return opts, scheduling, unpack(args) end function run(args) - local opts, conf_file, inv4_pcap, inv6_pcap = parse_args(args) + local opts, scheduling, conf_file, inv4_pcap, inv6_pcap = parse_args(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) if opts.name then engine.claim_name(opts.name) end local graph = config.new() if opts.reconfigurable then - setup.reconfigurable(setup.load_bench, graph, conf, + setup.reconfigurable(scheduling, setup.load_bench, graph, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') else + setup.apply_scheduling(scheduling) setup.load_bench(graph, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') end app.configure(graph) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 6d74f63755..bbf681730e 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -21,10 +21,9 @@ function parse_args(args) if #args == 0 then show_usage(1) end local conf_file, v4, v6 local ring_buffer_size - local opts = { - verbosity = 0, ingress_drop_monitor = 'flush', bench_file = 'bench.csv' } + local opts = { verbosity = 0, bench_file = 'bench.csv' } + local scheduling = { ingress_drop_monitor = 'flush' } local handlers = {} - local cpu function handlers.n (arg) opts.name = assert(arg) end function handlers.v () opts.verbosity = opts.verbosity + 1 end function handlers.i () opts.virtio_net = true end @@ -39,25 +38,14 @@ function parse_args(args) end end function handlers.cpu(arg) - cpu = tonumber(arg) + local cpu = tonumber(arg) if not cpu or cpu ~= math.floor(cpu) or cpu < 0 then fatal("Invalid cpu number: "..arg) end - - if opts.reconfigurable then - S.setenv("SNABB_TARGET_CPU", tostring(cpu), true) - local wanted_node = numa.cpu_get_numa_node(cpu) - numa.bind_to_numa_node(wanted_node) - print("Bound to numa node:", wanted_node) - else - print("Bound to CPU:", cpu) - numa.bind_to_cpu(cpu) - end + scheduling.cpu = cpu end handlers['real-time'] = function(arg) - if not S.sched_setscheduler(0, "fifo", 1) then - fatal('Failed to enable real-time scheduling. Try running as root.') - end + scheduling.real_time = true end function handlers.v4(arg) v4 = arg @@ -96,9 +84,9 @@ function parse_args(args) function handlers.b(arg) opts.bench_file = arg end handlers["ingress-drop-monitor"] = function (arg) if arg == 'flush' or arg == 'warn' then - opts.ingress_drop_monitor = arg + scheduling.ingress_drop_monitor = arg elseif arg == 'off' then - opts.ingress_drop_monitor = nil + scheduling.ingress_drop_monitor = false else fatal("invalid --ingress-drop-monitor argument: " .. arg .." (valid values: flush, warn, off)") @@ -122,15 +110,14 @@ function parse_args(args) if opts.mirror then assert(opts["on-a-stick"], "Mirror option is only valid in on-a-stick mode") end - if cpu then numa.bind_to_cpu(cpu) end if opts["on-a-stick"] then - numa.check_affinity_for_pci_addresses({ v4 }) - return opts, conf_file, v4 + scheduling.pci_addrs = { v4 } + return opts, scheduling, conf_file, v4 else - if not v4 then fatal("Missing required --v4-pci argument.") end - if not v6 then fatal("Missing required --v6-pci argument.") end - numa.check_affinity_for_pci_addresses({ v4, v6 }) - return opts, conf_file, v4, v6 + if not v4 then fatal("Missing required --v4 argument.") end + if not v6 then fatal("Missing required --v6 argument.") end + scheduling.pci_addrs = { v4, v6 } + return opts, scheduling, conf_file, v4, v6 end end @@ -143,7 +130,7 @@ local function requires_splitter (opts, conf) end function run(args) - local opts, conf_file, v4, v6 = parse_args(args) + local opts, scheduling, conf_file, v4, v6 = parse_args(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local use_splitter = requires_splitter(opts, conf) @@ -162,11 +149,14 @@ function run(args) else setup_fn, setup_args = setup.load_phy, { 'inetNic', v4, 'b4sideNic', v6 } end + if opts.reconfigurable then - setup.reconfigurable(setup_fn, c, conf, unpack(setup_args)) + setup.reconfigurable(scheduling, setup_fn, c, conf, unpack(setup_args)) else + setup.apply_scheduling(scheduling) setup_fn(c, conf, unpack(setup_args)) end + engine.configure(c) if opts.verbosity >= 2 then diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 956f11cb88..28015aa2c2 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -15,6 +15,7 @@ local pcap = require("apps.pcap.pcap") local ipv4_apps = require("apps.lwaftr.ipv4_apps") local ipv6_apps = require("apps.lwaftr.ipv6_apps") local vlan = require("apps.vlan.vlan") +local numa = require("lib.numa") local ipv4 = require("lib.protocol.ipv4") local ethernet = require("lib.protocol.ethernet") local ipv4_ntop = require("lib.yang.util").ipv4_ntop @@ -467,7 +468,60 @@ function load_soak_test_on_a_stick (c, conf, inv4_pcap, inv6_pcap) link_sink(c, unpack(sinks)) end -function reconfigurable(f, graph, conf, ...) +local apply_scheduling_opts = { + cpu = { required=false }, + pci_addrs = { default={} }, + real_time = { default=false }, + ingress_drop_monitor = { default='flush' } +} +function apply_scheduling(opts) + local lib = require("core.lib") + local ingress_drop_monitor = require("lib.timers.ingress_drop_monitor") + local fatal = require("apps.lwaftr.lwutil").fatal + + opts = lib.parse(opts, apply_scheduling_opts) + if opts.cpu then + numa.bind_to_cpu(opts.cpu) + print("Bound data plane to CPU:", opts.cpu) + end + numa.check_affinity_for_pci_addresses(opts.pci_addrs) + if opts.ingress_drop_monitor then + local mon = ingress_drop_monitor.new({action=opts.ingress_drop_monitor}) + timer.activate(mon:timer()) + end + if opts.real_time then + if not S.sched_setscheduler(0, "fifo", 1) then + fatal('Failed to enable real-time scheduling. Try running as root.') + end + end +end + +function run_worker(scheduling) + local app = require("core.app") + apply_scheduling(scheduling) + local myconf = config.new() + config.app(myconf, "follower", follower.Follower, {}) + app.configure(myconf) + app.busywait = true + app.main({}) +end + +local function stringify(x) + if type(x) == 'string' then return string.format('%q', x) end + if type(x) == 'number' then return tostring(x) end + if type(x) == 'boolean' then return x and 'true' or 'false' end + assert(type(x) == 'table') + local ret = {"{"} + local first = true + for k,v in pairs(x) do + if first then first = false else table.insert(ret, ",") end + table.insert(ret, string.format('[%s]=%s', stringify(k), stringify(v))) + end + table.insert(ret, "}") + return table.concat(ret) +end + +function reconfigurable(scheduling, f, graph, conf, ...) local args = {...} local function setup_fn(conf) local graph = config.new() @@ -475,23 +529,15 @@ function reconfigurable(f, graph, conf, ...) return graph end - local worker_code = string.format([[ - local follower = require("apps.config.follower") - local app = require("core.app") - local numa = require("lib.numa") + if scheduling.cpu then + local wanted_node = numa.cpu_get_numa_node(scheduling.cpu) + numa.bind_to_numa_node(wanted_node) + print("Bound main process to NUMA node: ", wanted_node) + end + + local worker_code = "require('program.lwaftr.setup').run_worker(%s)" + worker_code = worker_code:format(stringify(scheduling)) - local target_cpu = tonumber(%s) - if target_cpu then - numa.bind_to_cpu(target_cpu) - print("Bound worker to CPU: ", target_cpu) - end - local myconf = config.new() - config.app(myconf, "follower", follower.Follower, {}) - app.configure(myconf) - app.busywait = true - app.main({}) - ]], - S.getenv("SNABB_TARGET_CPU")) local follower_pid = worker.start("follower", worker_code) config.app(graph, 'leader', leader.Leader, From d3928a3065d1af886f25dc57a2ff037a06670c6e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 8 Dec 2016 11:03:45 +0100 Subject: [PATCH 409/631] Snabb lwAFTR v3.1.3 change log --- src/program/lwaftr/doc/CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index c9423cceb8..69ac289d7d 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,17 @@ # Change Log +## [3.1.3] - 2016-12-08 + + * Fix performance problem for --reconfigurable "snabb lwaftr run" + wherein the main coordination process would also get scheduled on the + data plane CPU. Also re-enable ingress drop monitor and --real-time + support for multiprocess lwaftr. + + * "snabb config --help" fixes. + + * Allow "snabb lwaftr query", "snabb lwaftr monitor", "snabbvmx query", + and "snabbvmx top" to locate Snabb instances by name. + ## [3.1.2] - 2016-12-07 * Re-enabled multi-process mode for --reconfigurable "snabb lwaftr From ae8d769fa97048dc97dd11dae98554bc2862821d Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 8 Dec 2016 11:18:30 +0100 Subject: [PATCH 410/631] Ship "snabb" binary and snabb-lwaftr symlink. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 31c20f9c98..adbf43780f 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,8 @@ dist: all mkdir "$(DISTDIR)" git clone "$(BUILDDIR)" "$(DISTDIR)/snabbswitch" rm -rf "$(DISTDIR)/snabbswitch/.git" - cp "$(BUILDDIR)/src/snabb" "$(DISTDIR)/$(DIST_BINARY)" + cp "$(BUILDDIR)/src/snabb" "$(DISTDIR)/" + if test "$(DIST_BINARY)" != "snabb"; then ln -s "snabb" "$(DISTDIR)/$(DIST_BINARY)"; fi cd "$(DISTDIR)/.." && tar cJvf "`basename '$(DISTDIR)'`.tar.xz" "`basename '$(DISTDIR)'`" rm -rf "$(DISTDIR)" From 87055ed830b0883e36fe09a83c21914685de1907 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 9 Dec 2016 02:26:35 +0100 Subject: [PATCH 411/631] Add tests for snabb config with lwaftr This adds a bunch of front-end tests that test verious quries using `snabb config [get|set|add|remove|get-state]`. Where it can it will validate the values it changes using the snabb config too (e.g. a snabb config set will check snabb config get holds the new value). --- .../lwaftr/tests/config/test-config-add.sh | 34 ++++++++++++ .../tests/config/test-config-get-state.sh | 35 ++++++++++++ .../lwaftr/tests/config/test-config-get.sh | 40 ++++++++++++++ .../lwaftr/tests/config/test-config-remove.sh | 34 ++++++++++++ .../lwaftr/tests/config/test-config-set.sh | 49 +++++++++++++++++ src/program/lwaftr/tests/config/tools.sh | 55 +++++++++++++++++++ 6 files changed, 247 insertions(+) create mode 100755 src/program/lwaftr/tests/config/test-config-add.sh create mode 100755 src/program/lwaftr/tests/config/test-config-get-state.sh create mode 100755 src/program/lwaftr/tests/config/test-config-get.sh create mode 100755 src/program/lwaftr/tests/config/test-config-remove.sh create mode 100755 src/program/lwaftr/tests/config/test-config-set.sh create mode 100755 src/program/lwaftr/tests/config/tools.sh diff --git a/src/program/lwaftr/tests/config/test-config-add.sh b/src/program/lwaftr/tests/config/test-config-add.sh new file mode 100755 index 0000000000..b1ff8153df --- /dev/null +++ b/src/program/lwaftr/tests/config/test-config-add.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +## This adds a softwire section and then checks it can be got +## back and that all the values are as they should be. + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +# Load the tools to be able to test stuff. +BASEDIR="`pwd`" +cd "`dirname \"$0\"`" +source tools.sh +cd $BASEDIR + +# Come up with a name for the lwaftr +SNABB_NAME="`random_name`" + +# Start the bench command. +start_lwaftr_bench $SNABB_NAME + +# IP to test with. +TEST_SOFTWIRE="{ ipv4 1.2.3.4; psid 72; b4-ipv6 ::1; br 1; }" +./snabb config add "$SNABB_NAME" "/softwire-config/binding-table/softwire" "$TEST_SOFTWIRE" + +# Check it can get this just fine +./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=1.2.3.4][psid=72] &> /dev/null +assert_equal $? 0 + +# Test that the b4-ipv4 is correct +ADDED_B4_IPV4="`./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=1.2.3.4][psid=72]/b4-ipv6`" +assert_equal "$ADDED_B4_IPV4" "::1" + +stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-get-state.sh b/src/program/lwaftr/tests/config/test-config-get-state.sh new file mode 100755 index 0000000000..12bf460f38 --- /dev/null +++ b/src/program/lwaftr/tests/config/test-config-get-state.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +## This makes verious quries to snabb config get-state to try and verify +## that it will run and produce values. The script has no way of +## validating the acuracy of the values but it'll check it works. + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +# Load the tools to be able to test stuff. +BASEDIR="`pwd`" +cd "`dirname \"$0\"`" +source tools.sh +cd $BASEDIR + +# Come up with a name for the lwaftr +SNABB_NAME="`random_name`" + +# Start the bench command. +start_lwaftr_bench $SNABB_NAME + +# Selecting a few at random which should have non-zero results +IN_IPV4="`snabb config get-state $SNABB_NAME /softwire-state/in-ipv4-bytes`" +if [[ "$IN_IPV4" == "0" ]]; then + produce_error "Counter should not show zero." +fi + +OUT_IPV4="`snabb config get-state $SNABB_NAME /softwire-state/out-ipv4-bytes`" +if [[ "$IN_IPV4" == "0" ]]; then + produce_error "Counter should not show zero." +fi + +./snabb config get-state "$SNABB_NAME" / +assert_equal "$?" "0" diff --git a/src/program/lwaftr/tests/config/test-config-get.sh b/src/program/lwaftr/tests/config/test-config-get.sh new file mode 100755 index 0000000000..64bc6394ab --- /dev/null +++ b/src/program/lwaftr/tests/config/test-config-get.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +## This tests querying from a known config, the test is obviously +## dependent on the values in the test data files used, however this +## allows for testing basic "getting". It performs numerous gets which +## on different paths. + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +# Load the tools to be able to test stuff. +BASEDIR="`pwd`" +cd "`dirname \"$0\"`" +source tools.sh +cd $BASEDIR + +# Come up with a name for the lwaftr +SNABB_NAME="`random_name`" + +# Start the bench command. +start_lwaftr_bench $SNABB_NAME + +# Check we can get a known value from the config. +INTERNAL_IP="`./snabb config get $SNABB_NAME /softwire-config/internal-interface/ip`" +assert_equal "$INTERNAL_IP" "8:9:a:b:c:d:e:f" + +EXTERNAL_IP="`./snabb config get $SNABB_NAME /softwire-config/external-interface/ip`" +assert_equal "$EXTERNAL_IP" "10.10.10.10" + +BT_B4_IPV6="`./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=178.79.150.233][psid=7850]/b4-ipv6`" +assert_equal "$BT_B4_IPV6" "127:11:12:13:14:15:16:128" + +# Finally test getting a value from the ietf-softwire schema +IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=127:22:33:44:55:66:77:128]/binding-ipv4-addr" +BINDING_IPV4="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" +assert_equal "$?" "0" +assert_equal "$BINDING_IPV4" "178.79.150.15" + +stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-remove.sh b/src/program/lwaftr/tests/config/test-config-remove.sh new file mode 100755 index 0000000000..657a4cf0b1 --- /dev/null +++ b/src/program/lwaftr/tests/config/test-config-remove.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +## This adds a softwire section and then checks it can be got +## back and that all the values are as they should be. + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +# Load the tools to be able to test stuff. +BASEDIR="`pwd`" +cd "`dirname \"$0\"`" +source tools.sh +cd $BASEDIR + +# Come up with a name for the lwaftr +SNABB_NAME="`random_name`" + +# Start the bench command. +start_lwaftr_bench $SNABB_NAME + +# Firstly lets verify that the thing we want to remove actually exists +./snabb config get "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null +assert_equal "$?" "0" + +# Then lets remove it +./snabb config remove "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null +assert_equal "$?" "0" + +# Then lets verify we can't find it +./snabb config get "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null || true +assert_equal "$?" "0" + +stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-set.sh b/src/program/lwaftr/tests/config/test-config-set.sh new file mode 100755 index 0000000000..818133179e --- /dev/null +++ b/src/program/lwaftr/tests/config/test-config-set.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +## This checks you can set values, it'll then perform a get to +## verify the value set is the value that is got too. + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +# Load the tools to be able to test stuff. +BASEDIR="`pwd`" +cd "`dirname \"$0\"`" +source tools.sh +cd $BASEDIR + +# Come up with a name for the lwaftr +SNABB_NAME="`random_name`" + +# Start the bench command. +start_lwaftr_bench $SNABB_NAME + +# IP to test with. +TEST_IPV4="208.118.235.148" +./snabb config set "$SNABB_NAME" "/softwire-config/external-interface/ip" "$TEST_IPV4" +SET_IP="`./snabb config get \"$SNABB_NAME\" \"/softwire-config/external-interface/ip\"`" +assert_equal "$SET_IP" "$TEST_IPV4" + +# Set a value in a list +TEST_IPV6="::1" +TEST_IPV4="178.79.150.15" +TEST_PSID="0" +./snabb config set "$SNABB_NAME" "/softwire-config/binding-table/softwire[ipv4=$TEST_IPV4][psid=$TEST_PSID]/b4-ipv6" "$TEST_IPV6" +SET_IP="`./snabb config get \"$SNABB_NAME\" \"/softwire-config/binding-table/softwire[ipv4=$TEST_IPV4][psid=$TEST_PSID]/b4-ipv6\"`" +assert_equal "$SET_IP" "$TEST_IPV6" + +# Check that the value above I just set is the same in the IETF schema +# We actually need to look this up backwards, lets just check the same +# IPv4 address was used as was used to set it above. +IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=::1]/binding-ipv4-addr" +IPV4_ADDR="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" +assert_equal "$IPV4_ADDR" "$TEST_IPV4" + +# Also check the portset, the IPv4 address alone isn't unique +IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=::1]/port-set/psid" +PSID="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" +assert_equal "$PSID" "$TEST_PSID" + +# Stop the lwaftr process. +stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/tools.sh b/src/program/lwaftr/tests/config/tools.sh new file mode 100755 index 0000000000..9cbeb3e466 --- /dev/null +++ b/src/program/lwaftr/tests/config/tools.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +function produce_error { + (>&2 echo $1) + exit 1 +} + +function random_name { + cat /dev/urandom | tr -dc 'a-z' | fold -w 20 | head -n 1 +} + + +# Takes two paremters and checks their equality. +# It takes an optional third argument which will +# be displayed if it fails the equality check. +# e.g. +# $ assert_equal "yellow "cat" -> error +# $ assert_equal "banana" "banana" -> nothing (valid) +function assert_equal { + if [[ -z "$2" ]]; then + produce_error "assert_equals: Not enough arguments." + exit 1 + fi + if [[ "$1" == "$2" ]]; then + return + else + if [[ "$3" == "" ]]; then + produce_error "Assert error: $1 != $2" + else + produce_error "Assert error: $3" + fi + fi +} + +# This starts the lwaftr process. The process should end when the script +# ends however, if something goes wrong and it doesn't end correctly, a +# duration is set to prevent it running indefinitely. +function start_lwaftr_bench { + ./snabb lwaftr bench --reconfigurable --bench-file /dev/null --name "$1" \ + --duration 20 \ + program/lwaftr/tests/data/icmp_on_fail.conf \ + program/lwaftr/tests/benchdata/ipv{4,6}-0550.pcap &> /dev/null & + + # This isn't ideal but it takes a little time for the lwaftr to properly start + # TODO: make this better? + sleep 2 +} + +function stop_lwaftr_bench { + # Get the job number for lwaftr bench + local jobid="`jobs | grep -i \"lwaftr bench\" | awk '{print $1}' | tr -d '[]+'`" + kill -15 "%$jobid" + # Wait until it's shutdown. + wait &> /dev/null +} From f9efd360520b0023a6f3373fb66f0b2777bb554f Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 9 Dec 2016 02:34:09 +0100 Subject: [PATCH 412/631] Stop the lwaftr after the get-state command too --- src/program/lwaftr/tests/config/test-config-get-state.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/program/lwaftr/tests/config/test-config-get-state.sh b/src/program/lwaftr/tests/config/test-config-get-state.sh index 12bf460f38..c256c988f3 100755 --- a/src/program/lwaftr/tests/config/test-config-get-state.sh +++ b/src/program/lwaftr/tests/config/test-config-get-state.sh @@ -33,3 +33,5 @@ fi ./snabb config get-state "$SNABB_NAME" / assert_equal "$?" "0" + +stop_lwaftr_bench From 93ef6bdbbd92eab4b96790f825cefec3988dc65a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 09:14:00 +0100 Subject: [PATCH 413/631] Fix killer bug in leader: write beyond array end Thanks to Asumu Takikawa for tracking this one down. Quoth http://luajit.org/ext_ffi_semantics.html in "Initializers": Byte arrays may also be initialized with a Lua string. This copies the whole string plus a terminating zero-byte. The copy stops early only if the array has a known, fixed size. I was thinking that ffi.new('uint8_t[?]', #str, str) would make an array with "known, fixed size". But I think that characteristic applies to the type, and is uint8_t[?] fixed, even when given its length? I thought so. I was wrong; you can't do this: local buf = ffi.typeof('uint8_t[?]', #str)(str) But you can do this: local buf = ffi.typeof('uint8_t[$]', #str)(str) Using the $ makes an array type with fixed size. The [?] makes a variable-length array (VLA). So this bug was that LuaJIT was writing a zero byte beyond the array, and that was corrupting the GC metadata (probably the header for the next object). --- src/apps/config/leader.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 53dcde7e93..0f7298257b 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -607,7 +607,7 @@ function Leader:handle_calls_from_peers() assert(type(reply) == 'string') reply = #reply..'\n'..reply peer.state = 'reply' - peer.buf = ffi.new('uint8_t[?]', #reply, reply) + peer.buf = ffi.new('uint8_t[?]', #reply+1, reply) peer.pos = 0 peer.len = #reply else From d1f45157aea12ae1931cf4342310fad976141638 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 09:22:57 +0100 Subject: [PATCH 414/631] Fix a couple more instances of the uint8_t[?] bug The fix is the same for action_codec. For listen, since the Lua string is protected already from GC, we can just cast it to const char*. --- src/apps/config/action_codec.lua | 4 +++- src/program/config/listen/listen.lua | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/apps/config/action_codec.lua b/src/apps/config/action_codec.lua index 6b53084166..cacccdad90 100644 --- a/src/apps/config/action_codec.lua +++ b/src/apps/config/action_codec.lua @@ -116,7 +116,9 @@ local function encoder() end function encoder:string(str) self:uint32(#str) - table.insert(self.out, ffi.new('uint8_t[?]', #str, str)) + local buf = ffi.new('uint8_t[?]', #str) + ffi.copy(buf, str, #str) + table.insert(self.out, buf) end function encoder:blob(blob) self:uint32(ffi.sizeof(blob)) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 009c11a6cd..d52d9ce456 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -151,7 +151,7 @@ local function buffered_output() function ret:write(str) table.insert(self.buf, str) end function ret:flush_to_fd(fd) local str = table.concat(self.buf) - local bytes = ffi.new('uint8_t[?]', #str, str) + local bytes = ffi.cast('const char*', str) local written = 0 while written < #str do local wrote = assert(S.write(fd, bytes + written, #str - written)) From e7267cc53157346c4be296c7764f3c240e227657 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 09:53:32 +0100 Subject: [PATCH 415/631] Snabb lwAFTR v3.1.4 change log --- src/program/lwaftr/doc/CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 69ac289d7d..74bb716609 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +## [3.1.4] - 2016-12-09 + + * Fix memory corruption bug in main process of --reconfigurable "snabb + lwaftr run" that would cause the dataplane to prematurely exit. + ## [3.1.3] - 2016-12-08 * Fix performance problem for --reconfigurable "snabb lwaftr run" From 38128e5dfd5df8052f256d1b971971c9d46e1352 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 8 Dec 2016 16:49:23 +0100 Subject: [PATCH 416/631] Extract json module from "snabb config listen" --- src/program/config/json.lua | 236 ++++++++++++++++++++++++++ src/program/config/listen/listen.lua | 242 +-------------------------- 2 files changed, 244 insertions(+), 234 deletions(-) create mode 100644 src/program/config/json.lua diff --git a/src/program/config/json.lua b/src/program/config/json.lua new file mode 100644 index 0000000000..4a59fd548d --- /dev/null +++ b/src/program/config/json.lua @@ -0,0 +1,236 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local S = require("syscall") +local ffi = require("ffi") + +-- A very limited json library that only does objects of strings, +-- designed to integrate well with poll(2) loops. + +function buffered_input_from_fd(fd) + local buf_size = 4096 + local buf = ffi.new('uint8_t[?]', buf_size) + local buf_end = 0 + local pos = 0 + local ret = {} + local eof = false + local function fill() + assert(pos == buf_end) + if eof then return 0 end + pos = 0 + buf_end = assert(S.read(fd, buf, buf_size)) + assert(0 <= buf_end and buf_end <= buf_size) + if buf_end == 0 then eof = true end + return buf_end + end + function ret:avail() return buf_end - pos end + function ret:getfd() return fd end + function ret:eof() return eof end + function ret:peek() + if pos == buf_end and fill() == 0 then return nil end + return string.char(buf[pos]) + end + function ret:discard() + assert(pos < buf_end) + pos = pos + 1 + end + return ret +end + +local whitespace_pat = '[ \n\r\t]' + +function drop_buffered_whitespace(input) + while input:avail() > 0 and input:peek():match(whitespace_pat) do + input:discard() + end +end + +local function take_while(input, pat) + local out = {} + while input:peek() and input:peek():match(pat) do + table.insert(out, input:peek()) + input:discard() + end + return table.concat(out) +end + +local function check(input, ch) + if input:peek() ~= ch then return false end + input:discard() + return true +end + +local function consume(input, ch) + if not check(input, ch) then + if input:eof() then error('unexpected EOF') end + error('expected '..ch..', got '..input:peek()) + end +end + +local function consume_pat(input, pat) + local ch = input:peek() + if ch == nil then error('unexpected EOF') end + if not ch:match(pat) then error('unexpected character '..ch) end + input:discard() + return ch +end + +local function skip_whitespace(input) take_while(input, whitespace_pat) end + +-- Pattern describing characters that can appear literally in a JSON +-- string. +local literal_string_chars_pat = '%w' +do + -- Printable non-alphanumeric ASCII chars, excluding control + -- characters, backslash, and double-quote. + local punctuation = "!#$%&'()*+,-./:;<=>?@[]^_`{|}~ " + for i=1,#punctuation do + local punctuation_pat = '%'..punctuation:sub(i,i) + literal_string_chars_pat = literal_string_chars_pat..punctuation_pat + end + literal_string_chars_pat = '['..literal_string_chars_pat..']' +end +-- The escapable characters in JSON. +local escaped_string_chars = + { r="\r", n="\n", t="\t", ["\\"]="\\", ['"']='"', b="\b", f="\f", ["/"]="/" } + +local function read_json_string(input) + consume(input, '"') + local parts = {} + while not check(input, '"') do + -- JSON strings support unicode. The encoding of the JSON could + -- be anything though UTF-8 is the likely one. Assume the + -- encoding is ASCII-compatible (like UTF-8) and restrict + -- ourselves to printable ASCII characters. + local part = take_while(input, literal_string_chars_pat) + if part == '' then + consume(input, "\\") + for k,v in pairs(escaped_string_chars) do + if check(input, k) then part = v; break end + end + if part == '' and check(input, "u") then + -- 4-hex-digit unicode escape. We only support ASCII + -- tho. + local hex = '0x' + for i=1,4 do hex = hex..consume_pat(input, "%x") end + local code = assert(tonumber(hex)) + if code >= 128 then error('non-ASCII character: \\u00'..hex) end + part = string.char(code) + end + end + table.insert(parts, part) + end + return table.concat(parts) +end + +function read_json_object(input) + consume(input, "{") + skip_whitespace(input) + local ret = {} + if not check(input, "}") then + repeat + skip_whitespace(input) + local k = read_json_string(input) + if ret[k] then error('duplicate key: '..k) end + skip_whitespace(input) + consume(input, ":") + skip_whitespace(input) + local v = read_json_string(input) + ret[k] = v + skip_whitespace(input) + until not check(input, ",") + skip_whitespace(input) + consume(input, "}") + end + return ret +end + +function buffered_output() + local ret = { buf = {} } + function ret:write(str) table.insert(self.buf, str) end + function ret:flush_to_fd(fd) + local str = table.concat(self.buf) + local bytes = ffi.cast('const char*', str) + local written = 0 + while written < #str do + local wrote = assert(S.write(fd, bytes + written, #str - written)) + written = written + wrote + end + end + return ret +end + +local function write_json_string(output, str) + output:write('"') + local pos = 1 + while pos <= #str do + local head = str:match('^('..literal_string_chars_pat..'+)', pos) + if head then + output:write(head) + pos = pos + #head + else + head = str:sub(pos, pos) + local escaped + for k,v in pairs(escaped_string_chars) do + if v == head then escaped = k; break end + end + if not escaped then + escaped = string.format("u00%.2x", head:byte(1)) + end + output:write('\\'..escaped) + pos = pos + 1 + end + end + output:write('"') +end + +function write_json_object(output, obj) + output:write('{') + local comma = false + for k,v in pairs(obj) do + if comma then output:write(',') else comma = true end + write_json_string(output, k) + output:write(':') + write_json_string(output, v) + end + output:write('}') +end + +function selftest () + print('selftest: program.config.json') + local equal = require('core.lib').equal + local function test_json(str, obj) + local tmp = os.tmpname() + local f = io.open(tmp, 'w') + f:write(str) + f:write(" ") -- whitespace sentinel on the end. + f:close() + for i = 1,2 do + local fd = S.open(tmp, 'rdonly') + local input = buffered_input_from_fd(fd) + local parsed = read_json_object(input) + assert(equal(parsed, obj)) + assert(not input:eof()) + assert(check(input, " ")) + assert(not input:peek()) + assert(input:eof()) + fd:close() + + local fd = assert(S.open(tmp, 'wronly, trunc')) + local output = buffered_output() + write_json_object(output, parsed) + output:write(' ') -- sentinel + output:flush_to_fd(fd) + fd:close() + end + os.remove(tmp) + end + test_json('{}', {}) + test_json('{"foo":"bar"}', {foo='bar'}) + test_json('{"foo":"bar","baz":"qux"}', {foo='bar', baz='qux'}) + test_json('{ "foo" : "bar" , "baz" : "qux" }', + {foo='bar', baz='qux'}) + test_json('{ "fo\\u000ao" : "ba\\r " , "baz" : "qux" }', + {['fo\no']='ba\r ', baz='qux'}) + print('selftest: ok') +end diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index d52d9ce456..ab0fd75c96 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -7,195 +7,7 @@ local rpc = require("lib.yang.rpc") local data = require("lib.yang.data") local path_lib = require("lib.yang.path") local common = require("program.config.common") - -local function buffered_input_from_fd(fd) - local buf_size = 4096 - local buf = ffi.new('uint8_t[?]', buf_size) - local buf_end = 0 - local pos = 0 - local ret = {} - local eof = false - local function fill() - assert(pos == buf_end) - if eof then return 0 end - pos = 0 - buf_end = assert(S.read(fd, buf, buf_size)) - assert(0 <= buf_end and buf_end <= buf_size) - if buf_end == 0 then eof = true end - return buf_end - end - function ret:avail() return buf_end - pos end - function ret:getfd() return fd end - function ret:eof() return eof end - function ret:peek() - if pos == buf_end and fill() == 0 then return nil end - return string.char(buf[pos]) - end - function ret:discard() - assert(pos < buf_end) - pos = pos + 1 - end - return ret -end - -local whitespace_pat = '[ \n\r\t]' - -local function drop_buffered_whitespace(input) - while input:avail() > 0 and input:peek():match(whitespace_pat) do - input:discard() - end -end - -local function take_while(input, pat) - local out = {} - while input:peek() and input:peek():match(pat) do - table.insert(out, input:peek()) - input:discard() - end - return table.concat(out) -end - -local function check(input, ch) - if input:peek() ~= ch then return false end - input:discard() - return true -end - -local function consume(input, ch) - if not check(input, ch) then - if input:eof() then error('unexpected EOF') end - error('expected '..ch..', got '..input:peek()) - end -end - -local function consume_pat(input, pat) - local ch = input:peek() - if ch == nil then error('unexpected EOF') end - if not ch:match(pat) then error('unexpected character '..ch) end - input:discard() - return ch -end - -local function skip_whitespace(input) take_while(input, whitespace_pat) end - --- Pattern describing characters that can appear literally in a JSON --- string. -local literal_string_chars_pat = '%w' -do - -- Printable non-alphanumeric ASCII chars, excluding control - -- characters, backslash, and double-quote. - local punctuation = "!#$%&'()*+,-./:;<=>?@[]^_`{|}~ " - for i=1,#punctuation do - local punctuation_pat = '%'..punctuation:sub(i,i) - literal_string_chars_pat = literal_string_chars_pat..punctuation_pat - end - literal_string_chars_pat = '['..literal_string_chars_pat..']' -end --- The escapable characters in JSON. -local escaped_string_chars = - { r="\r", n="\n", t="\t", ["\\"]="\\", ['"']='"', b="\b", f="\f", ["/"]="/" } - -local function read_json_string(input) - consume(input, '"') - local parts = {} - while not check(input, '"') do - -- JSON strings support unicode. The encoding of the JSON could - -- be anything though UTF-8 is the likely one. Assume the - -- encoding is ASCII-compatible (like UTF-8) and restrict - -- ourselves to printable ASCII characters. - local part = take_while(input, literal_string_chars_pat) - if part == '' then - consume(input, "\\") - for k,v in pairs(escaped_string_chars) do - if check(input, k) then part = v; break end - end - if part == '' and check(input, "u") then - -- 4-hex-digit unicode escape. We only support ASCII - -- tho. - local hex = '0x' - for i=1,4 do hex = hex..consume_pat(input, "%x") end - local code = assert(tonumber(hex)) - if code >= 128 then error('non-ASCII character: \\u00'..hex) end - part = string.char(code) - end - end - table.insert(parts, part) - end - return table.concat(parts) -end - -local function read_json_object(input) - consume(input, "{") - skip_whitespace(input) - local ret = {} - if not check(input, "}") then - repeat - skip_whitespace(input) - local k = read_json_string(input) - if ret[k] then error('duplicate key: '..k) end - skip_whitespace(input) - consume(input, ":") - skip_whitespace(input) - local v = read_json_string(input) - ret[k] = v - skip_whitespace(input) - until not check(input, ",") - skip_whitespace(input) - consume(input, "}") - end - return ret -end - -local function buffered_output() - local ret = { buf = {} } - function ret:write(str) table.insert(self.buf, str) end - function ret:flush_to_fd(fd) - local str = table.concat(self.buf) - local bytes = ffi.cast('const char*', str) - local written = 0 - while written < #str do - local wrote = assert(S.write(fd, bytes + written, #str - written)) - written = written + wrote - end - end - return ret -end - -local function write_json_string(output, str) - output:write('"') - local pos = 1 - while pos <= #str do - local head = str:match('^('..literal_string_chars_pat..'+)', pos) - if head then - output:write(head) - pos = pos + #head - else - head = str:sub(pos, pos) - local escaped - for k,v in pairs(escaped_string_chars) do - if v == head then escaped = k; break end - end - if not escaped then - escaped = string.format("u00%.2x", head:byte(1)) - end - output:write('\\'..escaped) - pos = pos + 1 - end - end - output:write('"') -end - -local function write_json_object(output, obj) - output:write('{') - local comma = false - for k,v in pairs(obj) do - if comma then output:write(',') else comma = true end - write_json_string(output, k) - output:write(':') - write_json_string(output, v) - end - output:write('}') -end +local json_lib = require("program.config.json") local function validate_value(schema_name, revision_date, path, value_str) local parser = common.data_parser(schema_name, path) @@ -232,14 +44,15 @@ function request_handlers.remove(schema_name, revision_date, path) end local function read_request(client, schema_name, revision_date) - local json = read_json_object(client) + local json = json_lib.read_json_object(client) local id, verb, path = assert(json.id), assert(json.verb), assert(json.path) path = path_lib.normalize_path(path) local handler = assert(request_handlers[data.normalize_id(verb)]) local req = handler(schema_name, revision_date, path, json.value) local function print_reply(reply) - local output = buffered_output() - write_json_object(output, {id=id, status='ok', value=reply.config}) + local output = json_lib.buffered_output() + json_lib.write_json_object(output, + {id=id, status='ok', value=reply.config}) output:flush_to_fd(1) -- stdout end return req, print_reply @@ -257,7 +70,7 @@ function run(args) local caller = rpc.prepare_caller('snabb-config-leader-v1') local leader = common.open_socket_or_die(args.instance_id) attach_listener(leader, caller, args.schema_name, args.revision_date) - local client = buffered_input_from_fd(0) -- stdin + local client = json_lib.buffered_input_from_fd(0) -- stdin local pollfds = S.types.t.pollfds({ {fd=leader, events="in"}, {fd=client, events="in"}}) @@ -294,12 +107,12 @@ function run(args) -- causes the buffer to fill, which itself shouldn't -- block given the IN flag in the revents.) client:peek() - drop_buffered_whitespace(client) + json_lib.drop_buffered_whitespace(client) end while client:avail() > 0 do local request, print_reply = read_request(client, args.schema_name, args.revision_date) - drop_buffered_whitespace(client) + json_lib.drop_buffered_whitespace(client) local msg, parse_reply = rpc.prepare_call( caller, request.method, request.args) local function have_reply(msg) @@ -315,42 +128,3 @@ function run(args) end end end - -function selftest () - print('selftest: program.config.listen.listen') - local equal = require('core.lib').equal - local function test_json(str, obj) - local tmp = os.tmpname() - local f = io.open(tmp, 'w') - f:write(str) - f:write(" ") -- whitespace sentinel on the end. - f:close() - for i = 1,2 do - local fd = S.open(tmp, 'rdonly') - local input = buffered_input_from_fd(fd) - local parsed = read_json_object(input) - assert(equal(parsed, obj)) - assert(not input:eof()) - assert(check(input, " ")) - assert(not input:peek()) - assert(input:eof()) - fd:close() - - local fd = assert(S.open(tmp, 'wronly, trunc')) - local output = buffered_output() - write_json_object(output, parsed) - output:write(' ') -- sentinel - output:flush_to_fd(fd) - fd:close() - end - os.remove(tmp) - end - test_json('{}', {}) - test_json('{"foo":"bar"}', {foo='bar'}) - test_json('{"foo":"bar","baz":"qux"}', {foo='bar', baz='qux'}) - test_json('{ "foo" : "bar" , "baz" : "qux" }', - {foo='bar', baz='qux'}) - test_json('{ "fo\\u000ao" : "ba\\r " , "baz" : "qux" }', - {['fo\no']='ba\r ', baz='qux'}) - print('selftest: ok') -end From d6a5dd981135f8c2427f1741c967bdb77d1acc0b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 8 Dec 2016 18:02:56 +0100 Subject: [PATCH 417/631] Add "snabb config bench" --- src/program/config/bench/README | 13 +++++ src/program/config/bench/README.inc | 1 + src/program/config/bench/bench.lua | 79 +++++++++++++++++++++++++++++ src/program/config/common.lua | 6 ++- src/program/config/json.lua | 2 +- 5 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 src/program/config/bench/README create mode 120000 src/program/config/bench/README.inc create mode 100644 src/program/config/bench/bench.lua diff --git a/src/program/config/bench/README b/src/program/config/bench/README new file mode 100644 index 0000000000..6a9588d759 --- /dev/null +++ b/src/program/config/bench/README @@ -0,0 +1,13 @@ +Usage: snabb config bench [OPTION]... INSTANCE FILE +Benchmark "snabb listen" by repeatedly feeding it commands from FILE. + +Available options: + -h, --help Display this message. + -s, --schema SCHEMA YANG data interface to request. + -r, --revision REVISION Require a specific revision of the YANG module. + +This command will fork off a "snabb config listen INSTANCE" child +process, passing it the -s and -r options. + +See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md +for full documentation. diff --git a/src/program/config/bench/README.inc b/src/program/config/bench/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/config/bench/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file diff --git a/src/program/config/bench/bench.lua b/src/program/config/bench/bench.lua new file mode 100644 index 0000000000..3fc857b3fc --- /dev/null +++ b/src/program/config/bench/bench.lua @@ -0,0 +1,79 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. +module(..., package.seeall) + +local S = require("syscall") +local common = require("program.config.common") +local json_lib = require("program.config.json") + +local function read_reply(fd) + local json = read_json_object(client) + local output = buffered_output() + write_json_object(output, json) + output:flush_to_fd(1) -- stdout +end + +local function read_commands(file) + local fd = assert(S.open(file, "rdonly")) + local input = json_lib.buffered_input_from_fd(fd:getfd()) + local ret = {} + json_lib.skip_whitespace(input) + while not input:eof() do + table.insert(ret, json_lib.read_json_object(input)) + json_lib.skip_whitespace(input) + end + fd:close() + return ret +end + +function die(input) + local chars = {} + while input:peek() do + table.insert(chars, input:peek()) + input:discard() + end + local str = table.concat(chars) + io.stderr:write("Error detected reading response:\n"..str) + main.exit(1) +end + +function run(args) + args, file = common.parse_command_line(args, { command='bench', + with_extra_args=1 }) + local commands = read_commands(file) + local ok, err, input_read, input_write = assert(S.pipe()) + local ok, err, output_read, output_write = assert(S.pipe()) + local pid = S.fork() + if pid == 0 then + local argv = {"snabb", "config", "listen", + "-s", args.schema_name, args.instance_id} + S.prctl("set_pdeathsig", "hup") + input_write:close() + output_read:close() + assert(S.dup2(input_read, 0)) + assert(S.dup2(output_write, 1)) + S.execve(("/proc/%d/exe"):format(S.getpid()), argv, {}) + end + input_read:close() + output_write:close() + + local input = json_lib.buffered_input_from_fd(output_read:getfd()) + local start = engine.now() + for _,json in ipairs(commands) do + local out = json_lib.buffered_output() + json_lib.write_json_object(out, json) + out:flush_to_fd(input_write) + json_lib.skip_whitespace(input) + local ok, response = pcall(json_lib.read_json_object, input) + if ok then + io.stdout:write(".") + io.stdout:flush() + else + die(input) + end + end + local elapsed = engine.now() - start + io.stdout:write("\n") + print(string.format("Issued %s commands in %.2f seconds (%.2f commands/s)", + #commands, elapsed, #commands/elapsed)) + main.exit(0) +end diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 30037cec2f..6cf46c1e52 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -21,6 +21,7 @@ local parse_command_line_opts = { with_config_file = { default=false }, with_path = { default=false }, with_value = { default=false }, + with_extra_args = { default=0 }, require_schema = { default=false } } @@ -76,8 +77,9 @@ function parse_command_line(args, opts) end ret.value = parser(ret.value_str) end - if #args ~= 0 then err("too many arguments") end - return ret + if #args < opts.with_extra_args then err("too few arguments") end + if #args ~= opts.with_extra_args then err("too many arguments") end + return ret, unpack(args) end function open_socket_or_die(instance_id) diff --git a/src/program/config/json.lua b/src/program/config/json.lua index 4a59fd548d..3723a85886 100644 --- a/src/program/config/json.lua +++ b/src/program/config/json.lua @@ -75,7 +75,7 @@ local function consume_pat(input, pat) return ch end -local function skip_whitespace(input) take_while(input, whitespace_pat) end +function skip_whitespace(input) take_while(input, whitespace_pat) end -- Pattern describing characters that can appear literally in a JSON -- string. From 4ea7fdc603bb19e431dc826033b7d4857a65ab03 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 8 Dec 2016 19:01:00 +0100 Subject: [PATCH 418/631] Tighten up ctable interation bounds This avoids a number of cases where we assume that size*2 entries are available to traverse. Bump the binary version too so that people with compiled configs have to update. --- src/lib/ctable.lua | 17 +++++++---------- src/lib/yang/binary.lua | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index fd13968f7f..294dbdc1fc 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -194,7 +194,7 @@ function load(stream, params) local ctab = new(params_copy) ctab.occupancy = header.occupancy ctab.max_displacement = header.max_displacement - local entry_count = ctab.size + ctab.max_displacement + 1 + local entry_count = ctab.size + ctab.max_displacement -- Slurp the entries directly into the ctable's backing store. -- This ensures that the ctable is in hugepages. @@ -211,7 +211,7 @@ function CTable:save(stream) header_t) stream:write_array(self.entries, self.entry_type, - self.size + self.max_displacement + 1) + self.size + self.max_displacement) end function CTable:insert(hash, key, value, updates_allowed) @@ -321,7 +321,7 @@ function CTable:remove_ptr(entry) local scale = self.scale local index = entry - self.entries assert(index >= 0) - assert(index <= self.size + self.max_displacement) + assert(index < self.size + self.max_displacement) assert(entry.hash ~= HASH_MAX) self.occupancy = self.occupancy - 1 @@ -466,7 +466,7 @@ function CTable:selfcheck() end local prev = 0 - for i = 0,self.size*2-1 do + for i = 0,self.size+self.max_displacement-1 do local entry = self.entries[i] local hash = entry.hash if hash ~= 0xffffffff then @@ -501,18 +501,15 @@ function CTable:dump() io.write(' value: '..tostring(entry.value)..'\n') end end - for index=0,self.size-1 do dump_one(index) end - for index=self.size,self.size*2-1 do - if self.entries[index].hash == HASH_MAX then break end - dump_one(index) - end + for index=0,self.size-1+self.max_displacement do dump_one(index) end end function CTable:iterate() local max_entry = self.entries + self.size + self.max_displacement local function next_entry(max_entry, entry) - while entry <= max_entry do + while true do entry = entry + 1 + if entry >= max_entry then return nil end if entry.hash ~= HASH_MAX then return entry end end end diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index fd7368d094..f9d80028be 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -12,7 +12,7 @@ local ctable = require('lib.ctable') local cltable = require('lib.cltable') local MAGIC = "yangconf" -local VERSION = 0x00001000 +local VERSION = 0x00002000 local header_t = ffi.typeof([[ struct { From 6ff54ff55b3c25d93c09c2fb39b55a82d71d9eb6 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 09:16:37 +0100 Subject: [PATCH 419/631] Memoize schema loads --- src/lib/yang/schema.lua | 2 ++ src/lib/yang/util.lua | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/lib/yang/schema.lua b/src/lib/yang/schema.lua index a28bbe40ff..ebeb2babcf 100644 --- a/src/lib/yang/schema.lua +++ b/src/lib/yang/schema.lua @@ -868,12 +868,14 @@ function load_schema_file(filename) local s, e = resolve(primitivize(parse_schema_file(filename))) return inherit_config(s), e end +load_schema_file = util.memoize(load_schema_file) function load_schema_by_name(name, revision) -- FIXME: @ is not valid in a Lua module name. -- if revision then name = name .. '@' .. revision end name = name:gsub('-', '_') return load_schema(require('lib.yang.'..name..'_yang'), name) end +load_schema_by_name = util.memoize(load_schema_by_name) function selftest() print('selftest: lib.yang.schema') diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 91e8ff17c5..62e60e6370 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -95,6 +95,19 @@ function string_output_file() return file end +function memoize(f) + local cache = {} + return function(...) + local args = {...} + for k,v in pairs(cache) do + if lib.equal(k, args) then return unpack(v) end + end + local ret = {f(...)} + cache[args] = ret + return unpack(ret) + end +end + function selftest() print('selftest: lib.yang.util') assert(tointeger('0') == 0) From 73fc4eaa3f30998b6099d6c00875a5522ad3d31c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 09:50:53 +0100 Subject: [PATCH 420/631] Add some more memoizations --- src/apps/config/leader.lua | 1 + src/lib/yang/path.lua | 1 + 2 files changed, 2 insertions(+) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 0f7298257b..993f432253 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -308,6 +308,7 @@ end function compute_add_config_fn (schema_name, path) return path_adder_for_schema(yang.load_schema_by_name(schema_name), path) end +compute_add_config_fn = util.memoize(compute_add_config_fn) local function path_remover_for_grammar(grammar, path) local top_grammar = grammar diff --git a/src/lib/yang/path.lua b/src/lib/yang/path.lua index cb656643d5..26e7f16edb 100644 --- a/src/lib/yang/path.lua +++ b/src/lib/yang/path.lua @@ -234,6 +234,7 @@ function resolver(grammar, path_string) end return getter, grammar end +resolver = util.memoize(resolver) -- Loads a module and converts the rest of the path. function load_from_path(path) From 84a0ac1473832ebd5df111b7caffb1c90031c44a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 9 Dec 2016 10:28:36 +0100 Subject: [PATCH 421/631] Add forgotten selftest.sh --- src/program/lwaftr/tests/config/selftest.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 src/program/lwaftr/tests/config/selftest.sh diff --git a/src/program/lwaftr/tests/config/selftest.sh b/src/program/lwaftr/tests/config/selftest.sh new file mode 100755 index 0000000000..c9ffba7066 --- /dev/null +++ b/src/program/lwaftr/tests/config/selftest.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -o errexit + +./program/lwaftr/tests/config/test-config-get.sh +./program/lwaftr/tests/config/test-config-set.sh +./program/lwaftr/tests/config/test-config-add.sh +./program/lwaftr/tests/config/test-config-remove.sh +./program/lwaftr/tests/config/test-config-get-state.sh From 6e0cd4d9e8f9fa229b10b1a543d84c9ce03a64b9 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 9 Dec 2016 10:29:56 +0100 Subject: [PATCH 422/631] Remove TODO where solution was acceptable --- src/program/lwaftr/tests/config/tools.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/program/lwaftr/tests/config/tools.sh b/src/program/lwaftr/tests/config/tools.sh index 9cbeb3e466..b55dac446f 100755 --- a/src/program/lwaftr/tests/config/tools.sh +++ b/src/program/lwaftr/tests/config/tools.sh @@ -42,7 +42,6 @@ function start_lwaftr_bench { program/lwaftr/tests/benchdata/ipv{4,6}-0550.pcap &> /dev/null & # This isn't ideal but it takes a little time for the lwaftr to properly start - # TODO: make this better? sleep 2 } From a9a4a55217bf9b1c0ec85d239be528539d733ca0 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 10:58:44 +0100 Subject: [PATCH 423/631] Inline "snabb config bench" args parsing --- src/program/config/bench/bench.lua | 39 ++++++++++++++++++++++++++---- src/program/config/common.lua | 6 ++--- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/program/config/bench/bench.lua b/src/program/config/bench/bench.lua index 3fc857b3fc..44de820095 100644 --- a/src/program/config/bench/bench.lua +++ b/src/program/config/bench/bench.lua @@ -2,9 +2,31 @@ module(..., package.seeall) local S = require("syscall") -local common = require("program.config.common") +local lib = require("core.lib") local json_lib = require("program.config.json") +function show_usage(command, status, err_msg) + if err_msg then print('error: '..err_msg) end + print(require("program.config.bench.README_inc")) + main.exit(status) +end + +function parse_command_line(args) + local function err(msg) show_usage(1, msg) end + local listen_params = {} + local handlers = {} + function handlers.h() show_usage(0) end + function handlers.s(arg) listen_params.schema_name = arg end + function handlers.r(arg) listen_params.revision_date = arg end + args = lib.dogetopt(args, handlers, "hs:r:", + {help="h", ['schema-name']="s", schema="s", + ['revision-date']="r", revision="r"}) + if #args ~= 2 then err() end + local commands_file + listen_params.instance_id, commands_file = unpack(args) + return listen_params, commands_file +end + local function read_reply(fd) local json = read_json_object(client) local output = buffered_output() @@ -37,15 +59,22 @@ function die(input) end function run(args) - args, file = common.parse_command_line(args, { command='bench', - with_extra_args=1 }) + listen_params, file = parse_command_line(args) local commands = read_commands(file) local ok, err, input_read, input_write = assert(S.pipe()) local ok, err, output_read, output_write = assert(S.pipe()) local pid = S.fork() if pid == 0 then - local argv = {"snabb", "config", "listen", - "-s", args.schema_name, args.instance_id} + local argv = {"snabb", "config", "listen"} + if listen_params.schema_name then + table.insert(argv, "-s") + table.insert(argv, listen_params.schema_name) + end + if listen_params.revision_date then + table.insert(argv, "-r") + table.insert(argv, listen_params.revision_date) + end + table.insert(argv, listen_params.instance_id) S.prctl("set_pdeathsig", "hup") input_write:close() output_read:close() diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 6cf46c1e52..30037cec2f 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -21,7 +21,6 @@ local parse_command_line_opts = { with_config_file = { default=false }, with_path = { default=false }, with_value = { default=false }, - with_extra_args = { default=0 }, require_schema = { default=false } } @@ -77,9 +76,8 @@ function parse_command_line(args, opts) end ret.value = parser(ret.value_str) end - if #args < opts.with_extra_args then err("too few arguments") end - if #args ~= opts.with_extra_args then err("too many arguments") end - return ret, unpack(args) + if #args ~= 0 then err("too many arguments") end + return ret end function open_socket_or_die(instance_id) From d27a5dfb02d5936c0d499488559a3868e717ed2b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 11:40:42 +0100 Subject: [PATCH 424/631] Add pipelining to "snabb config bench" Also change json lib to use wrapped fd's. --- src/program/config/bench/bench.lua | 58 ++++++++++++++++++++-------- src/program/config/json.lua | 13 ++++--- src/program/config/listen/listen.lua | 4 +- 3 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/program/config/bench/bench.lua b/src/program/config/bench/bench.lua index 44de820095..283c2a20ab 100644 --- a/src/program/config/bench/bench.lua +++ b/src/program/config/bench/bench.lua @@ -3,6 +3,7 @@ module(..., package.seeall) local S = require("syscall") local lib = require("core.lib") +local ffi = require("ffi") local json_lib = require("program.config.json") function show_usage(command, status, err_msg) @@ -31,17 +32,20 @@ local function read_reply(fd) local json = read_json_object(client) local output = buffered_output() write_json_object(output, json) - output:flush_to_fd(1) -- stdout + output:flush(S.stdout) end local function read_commands(file) local fd = assert(S.open(file, "rdonly")) - local input = json_lib.buffered_input_from_fd(fd:getfd()) - local ret = {} + local input = json_lib.buffered_input(fd) json_lib.skip_whitespace(input) + local ret = {} while not input:eof() do - table.insert(ret, json_lib.read_json_object(input)) + local json = json_lib.read_json_object(input) json_lib.skip_whitespace(input) + local out = json_lib.buffered_output() + json_lib.write_json_object(out, json) + table.insert(ret, out:flush()) end fd:close() return ret @@ -58,6 +62,15 @@ function die(input) main.exit(1) end +function full_write(fd, str) + local ptr = ffi.cast("const char*", str) + local written = 0 + while written < #str do + local count = assert(fd:write(ptr + written, #str - written)) + written = written + count + end +end + function run(args) listen_params, file = parse_command_line(args) local commands = read_commands(file) @@ -85,19 +98,32 @@ function run(args) input_read:close() output_write:close() - local input = json_lib.buffered_input_from_fd(output_read:getfd()) + local write_buffering = assert(input_write:fcntl(S.c.F.GETPIPE_SZ)) + + local input = json_lib.buffered_input(output_read) local start = engine.now() - for _,json in ipairs(commands) do - local out = json_lib.buffered_output() - json_lib.write_json_object(out, json) - out:flush_to_fd(input_write) - json_lib.skip_whitespace(input) - local ok, response = pcall(json_lib.read_json_object, input) - if ok then - io.stdout:write(".") - io.stdout:flush() - else - die(input) + local next_write, next_read = 1, 1 + local buffered_bytes = 0 + io.stdout:setvbuf("no") + while next_read <= #commands do + while next_write <= #commands do + local str = commands[next_write] + if buffered_bytes + #str > write_buffering then break end + full_write(input_write, str) + io.stdout:write("w") + buffered_bytes = buffered_bytes + #str + next_write = next_write + 1 + end + while next_read < next_write do + json_lib.skip_whitespace(input) + local ok, response = pcall(json_lib.read_json_object, input) + if ok then + buffered_bytes = buffered_bytes - #commands[next_read] + next_read = next_read + 1 + io.stdout:write("r") + else + die(input) + end end end local elapsed = engine.now() - start diff --git a/src/program/config/json.lua b/src/program/config/json.lua index 3723a85886..a5ecaa380a 100644 --- a/src/program/config/json.lua +++ b/src/program/config/json.lua @@ -7,7 +7,7 @@ local ffi = require("ffi") -- A very limited json library that only does objects of strings, -- designed to integrate well with poll(2) loops. -function buffered_input_from_fd(fd) +function buffered_input(fd) local buf_size = 4096 local buf = ffi.new('uint8_t[?]', buf_size) local buf_end = 0 @@ -18,13 +18,13 @@ function buffered_input_from_fd(fd) assert(pos == buf_end) if eof then return 0 end pos = 0 - buf_end = assert(S.read(fd, buf, buf_size)) + buf_end = assert(fd:read(buf, buf_size)) assert(0 <= buf_end and buf_end <= buf_size) if buf_end == 0 then eof = true end return buf_end end function ret:avail() return buf_end - pos end - function ret:getfd() return fd end + function ret:getfd() return fd:getfd() end function ret:eof() return eof end function ret:peek() if pos == buf_end and fill() == 0 then return nil end @@ -148,12 +148,13 @@ end function buffered_output() local ret = { buf = {} } function ret:write(str) table.insert(self.buf, str) end - function ret:flush_to_fd(fd) + function ret:flush(fd) local str = table.concat(self.buf) + if fd == nil then return str end local bytes = ffi.cast('const char*', str) local written = 0 while written < #str do - local wrote = assert(S.write(fd, bytes + written, #str - written)) + local wrote = assert(fd:write(bytes + written, #str - written)) written = written + wrote end end @@ -207,7 +208,7 @@ function selftest () f:close() for i = 1,2 do local fd = S.open(tmp, 'rdonly') - local input = buffered_input_from_fd(fd) + local input = buffered_input(fd) local parsed = read_json_object(input) assert(equal(parsed, obj)) assert(not input:eof()) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index ab0fd75c96..28739af12a 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -53,7 +53,7 @@ local function read_request(client, schema_name, revision_date) local output = json_lib.buffered_output() json_lib.write_json_object(output, {id=id, status='ok', value=reply.config}) - output:flush_to_fd(1) -- stdout + output:flush(S.stdout) end return req, print_reply end @@ -70,7 +70,7 @@ function run(args) local caller = rpc.prepare_caller('snabb-config-leader-v1') local leader = common.open_socket_or_die(args.instance_id) attach_listener(leader, caller, args.schema_name, args.revision_date) - local client = json_lib.buffered_input_from_fd(0) -- stdin + local client = json_lib.buffered_input(S.stdin) local pollfds = S.types.t.pollfds({ {fd=leader, events="in"}, {fd=client, events="in"}}) From a2ae3b92a94f5641922ed7a5a1339d3c3bfe2dcf Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 11:55:31 +0100 Subject: [PATCH 425/631] Leader drains pending input from client Allow the leader to handle multiple requests from a client in one tick. --- src/apps/config/leader.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 993f432253..cc5d58b18b 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -548,6 +548,7 @@ function Leader:handle_calls_from_peers() local i = 1 while i <= #peers do local peer = peers[i] + local visit_peer_again = false while peer.state == 'length' do local ch, err = peer.fd:read(nil, 1) if not ch then @@ -618,6 +619,7 @@ function Leader:handle_calls_from_peers() end while peer.state == 'reply' do if peer.pos == peer.len then + visit_peer_again = true peer.state = 'length' peer.buf, peer.pos = nil, nil peer.len = 0 @@ -642,7 +644,7 @@ function Leader:handle_calls_from_peers() peer.fd:close() table.remove(peers, i) if self.listen_peer == peer then self.listen_peer = nil end - else + elseif not visit_peer_again then i = i + 1 end end From efabf7f3ef2901e5f82a35b13ffa4b7e15679209 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 11:57:31 +0100 Subject: [PATCH 426/631] Fix json lib for flush_to_fd rename --- src/program/config/json.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/json.lua b/src/program/config/json.lua index a5ecaa380a..53aa40a11d 100644 --- a/src/program/config/json.lua +++ b/src/program/config/json.lua @@ -221,7 +221,7 @@ function selftest () local output = buffered_output() write_json_object(output, parsed) output:write(' ') -- sentinel - output:flush_to_fd(fd) + output:flush(fd) fd:close() end os.remove(tmp) From 73e6be9e4b10a224319fc8b676e86c8ce005d035 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 8 Dec 2016 17:50:48 +0000 Subject: [PATCH 427/631] Exit nicely if couldn't set affinity for a core Setting affinity for a core can fail if trying to set affinity for a non-existent core, for instance. Remove use of not defined function 'fatal' in lib/numa.lib. --- src/lib/numa.lua | 8 ++++---- src/program/lwaftr/setup.lua | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/numa.lua b/src/lib/numa.lua index 3eeac22c8d..0bedf65ab1 100644 --- a/src/lib/numa.lua +++ b/src/lib/numa.lua @@ -92,7 +92,8 @@ function bind_to_cpu (cpu) if not cpu then return unbind_cpu() end assert(not bound_cpu, "already bound") - assert(S.sched_setaffinity(0, cpu)) + assert(S.sched_setaffinity(0, cpu), + ("Couldn't set affinity for cpu %s"):format(cpu)) local cpu_and_node = S.getcpu() assert(cpu_and_node.cpu == cpu) bound_cpu = cpu @@ -120,9 +121,8 @@ function bind_to_numa_node (node) end function prevent_preemption(priority) - if not S.sched_setscheduler(0, "fifo", priority or 1) then - fatal('Failed to enable real-time scheduling. Try running as root.') - end + assert(S.sched_setscheduler(0, "fifo", priority or 1), + 'Failed to enable real-time scheduling. Try running as root.') end function selftest () diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 28015aa2c2..ca0cda0639 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -481,7 +481,8 @@ function apply_scheduling(opts) opts = lib.parse(opts, apply_scheduling_opts) if opts.cpu then - numa.bind_to_cpu(opts.cpu) + local success, err = pcall(numa.bind_to_cpu, opts.cpu) + if not success then fatal(err) end print("Bound data plane to CPU:", opts.cpu) end numa.check_affinity_for_pci_addresses(opts.pci_addrs) From 655b799290abff062aac9a17e5599fe129883dd3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 12:14:38 +0100 Subject: [PATCH 428/631] Improve memoizer; memoize more yang functions --- src/lib/yang/data.lua | 3 +++ src/lib/yang/util.lua | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 6ba3c1cd97..8d71a7bd7d 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -116,6 +116,7 @@ function data_grammar_from_schema(schema) local members = visit_body(schema) return {type="struct", members=members, ctype=struct_ctype(members)} end +data_grammar_from_schema = util.memoize(data_grammar_from_schema) function rpc_grammar_from_schema(schema) local grammar = {} @@ -458,6 +459,7 @@ function data_parser_from_grammar(production) end return assert(top_parsers[production.type])(production) end +data_parser_from_grammar = util.memoize(data_parser_from_grammar) function load_data_for_schema(schema, str, filename) return data_parser_from_schema(schema)(str, filename) @@ -670,6 +672,7 @@ function data_printer_from_grammar(production) end return assert(top_printers[production.type])(production) end +data_printer_from_grammar = util.memoize(data_printer_from_grammar) local function string_output_file() local file = {} diff --git a/src/lib/yang/util.lua b/src/lib/yang/util.lua index 62e60e6370..149c900938 100644 --- a/src/lib/yang/util.lua +++ b/src/lib/yang/util.lua @@ -95,15 +95,35 @@ function string_output_file() return file end -function memoize(f) +function memoize(f, max_occupancy) local cache = {} + local occupancy = 0 + local argc = 0 + max_occupancy = max_occupancy or 10 return function(...) local args = {...} - for k,v in pairs(cache) do - if lib.equal(k, args) then return unpack(v) end + if #args == argc then + local walk = cache + for i=1,#args do + if walk == nil then break end + walk = walk[args[i]] + end + if walk ~= nil then return unpack(walk) end + else + cache, occupancy, argc = {}, 0, #args end local ret = {f(...)} - cache[args] = ret + if occupancy >= max_occupancy then + cache = {} + occupancy = 0 + end + local walk = cache + for i=1,#args-1 do + if not walk[args[i]] then walk[args[i]] = {} end + walk = walk[args[i]] + end + walk[args[#args]] = ret + occupancy = occupancy + 1 return unpack(ret) end end From d1f192c9c97091ff1e636afe1e2152b4c1663e37 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 9 Dec 2016 12:53:20 +0100 Subject: [PATCH 429/631] Remove garbled lwaftr-virt configuration file The file was garbled. In addition, there's already lwaftrctl.conf.example that works as an example configuration file. --- src/program/lwaftr/virt/confs/lwaftrctl1.conf | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 src/program/lwaftr/virt/confs/lwaftrctl1.conf diff --git a/src/program/lwaftr/virt/confs/lwaftrctl1.conf b/src/program/lwaftr/virt/confs/lwaftrctl1.conf deleted file mode 100644 index 5d945217cd..0000000000 --- a/src/program/lwaftr/virt/confs/lwaftrctl1.conf +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - ^ -lib/yang/parser.lua:44: :1:2: error: Invalid identifier -stack traceback: - core/main.lua:137: in function - [C]: in function 'error' - lib/yang/parser.lua:44: in function 'error' - lib/yang/parser.lua:195: in function 'parse_identifier' - lib/yang/parser.lua:208: in function 'parse_keyword' - lib/yang/parser.lua:243: in function 'parse_statement' - lib/yang/parser.lua:232: in function 'parse_statement_list' - lib/yang/parser.lua:220: in function 'parse_string' - lib/yang/data.lua:332: in function 'load_data_for_schema_by_name' - lib/yang/yang.lua:107: in function 'load_configuration' - ...m/lwaftr/migrate_configuration/migrate_configuration.lua:350: in function 'run' - program/lwaftr/lwaftr.lua:17: in function 'run' - core/main.lua:56: in function - [C]: in function 'xpcall' - core/main.lua:185: in main chunk - [C]: at 0x00453c60 - [C]: in function 'pcall' - core/startup.lua:3: in main chunk - [C]: in function 'require' - [string "require "core.startup""]:1: in main chunk From ba73125e19d553fcc2de702a2a8911322ab070c2 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 13:52:09 +0100 Subject: [PATCH 430/631] Be resilient to binary compilation version changes --- src/lib/yang/yang.lua | 45 +++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/lib/yang/yang.lua b/src/lib/yang/yang.lua index 2e3cc22bba..7727ac9d9e 100644 --- a/src/lib/yang/yang.lua +++ b/src/lib/yang/yang.lua @@ -63,19 +63,34 @@ function load_configuration(filename, opts) local function is_fresh(expected, got) end local function load_compiled(stream, source_mtime) - local compiled = binary.load_compiled_data(stream) - if opts.schema_name then - expect(opts.schema_name, compiled.schema_name, 'schema name') + local ok, result = pcall(binary.load_compiled_data, stream) + if not ok then + log('failed to load compiled configuration: %s', tostring(result)) + return end - if opts.revision_date then - expect(opts.revision_date, compiled.revision_date, - 'schema revision date') + local compiled = result + if opts.schema_name and opts.schema_name ~= compiled.schema_name then + log('expected schema name %s in compiled file, but got %s', + opts.schema_name, compiled.schema.name) + return end - if source_mtime == nil or - (source_mtime.sec == compiled.source_mtime.sec and - source_mtime.nsec == compiled.source_mtime.nsec) then - return compiled.data + if opts.revision_date and opts.revision_date ~= schema.revision_date then + log('expected schema revision date %s in compiled file, but got %s', + opts.revision_date, compiled.revision_date) + return end + if source_mtime then + if (source_mtime.sec == compiled.source_mtime.sec and + source_mtime.nsec == compiled.source_mtime.nsec) then + log('compiled configuration is up to date.') + return compiled.data + end + log('compiled configuration is out of date; recompiling.') + return + end + -- No source file. + log('loaded compiled configuration with no corresponding source file.') + return compiled.data end local source = stream.open_input_byte_stream(filename) @@ -92,12 +107,7 @@ function load_configuration(filename, opts) if binary.has_magic(compiled_stream) then log('loading compiled configuration from %s', compiled_filename) local conf = load_compiled(compiled_stream, source_mtime) - if conf then - log('compiled configuration %s is up to date.', compiled_filename) - return conf - end - log('compiled configuration %s is out of date; recompiling.', - compiled_filename) + if conf then return conf end end compiled_stream:close() end @@ -105,7 +115,7 @@ function load_configuration(filename, opts) -- Load and compile it. local source_str = source:read_string() source:close() - log('loading source configuration from %s', filename) + log('loading source configuration') local conf = load_data_for_schema_by_name(opts.schema_name, source_str, filename) @@ -117,6 +127,7 @@ function load_configuration(filename, opts) log('error saving compiled configuration %s: %s', compiled_filename, err) end + log('wrote compiled configuration %s', compiled_filename) -- Done. return conf end From 041b0e41db905320a40f0586ceaa94208cb1d7e6 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 14:45:06 +0100 Subject: [PATCH 431/631] Removed README. prefix from docs --- src/program/lwaftr/doc/CHANGELOG.md | 10 +++---- src/program/lwaftr/doc/README.md | 20 +++++++------- ...README.benchmarking.md => benchmarking.md} | 6 ++--- ...ADME.configuration.md => configuration.md} | 2 +- ...tegration.md => continuous-integration.md} | 0 .../doc/{README.counters.md => counters.md} | 2 +- ...-performance.md => filters-performance.md} | 0 src/program/lwaftr/doc/genbook.sh | 26 +++++++++---------- .../lwaftr/doc/{README.ndp.md => ndp.md} | 0 .../{README.performance.md => performance.md} | 0 ...ADME.rfccompliance.md => rfccompliance.md} | 0 .../doc/{README.running.md => running.md} | 6 ++--- ....troubleshooting.md => troubleshooting.md} | 0 ...ME.virtualization.md => virtualization.md} | 0 14 files changed, 36 insertions(+), 36 deletions(-) rename src/program/lwaftr/doc/{README.benchmarking.md => benchmarking.md} (94%) rename src/program/lwaftr/doc/{README.configuration.md => configuration.md} (99%) rename src/program/lwaftr/doc/{README.continuous-integration.md => continuous-integration.md} (100%) rename src/program/lwaftr/doc/{README.counters.md => counters.md} (99%) rename src/program/lwaftr/doc/{README.filters-performance.md => filters-performance.md} (100%) rename src/program/lwaftr/doc/{README.ndp.md => ndp.md} (100%) rename src/program/lwaftr/doc/{README.performance.md => performance.md} (100%) rename src/program/lwaftr/doc/{README.rfccompliance.md => rfccompliance.md} (100%) rename src/program/lwaftr/doc/{README.running.md => running.md} (94%) rename src/program/lwaftr/doc/{README.troubleshooting.md => troubleshooting.md} (100%) rename src/program/lwaftr/doc/{README.virtualization.md => virtualization.md} (100%) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 74bb716609..e6c1f6ae15 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -105,7 +105,7 @@ A change to migrate the lwAFTR to use a new YANG-based configuration. configurations, run "snabb lwaftr migrate-configation old.conf" on the old configuration. See the [snabb-softwire-v1.yang schema](../../../lib/yang/snabb-softwire-v1.yang) or - [README.configuration.md](./README.configuration.md) for full details + [configuration.md](./configuration.md) for full details on the new configuration format. * Send ICMPv6 unreachable messages from the most appropriate source address @@ -119,7 +119,7 @@ A change to migrate the lwAFTR to use a new YANG-based configuration. `snabbvmx` documentation](../../snabbvmx/doc/README.md) for more. * Add many more counters, used to diagnose the path that packets take - in the lwAFTR. See [README.counters.md](./README.counters.md) for + in the lwAFTR. See [counters.md](./counters.md) for more. * Add "snabb config" set of commands, to replace "snabb lwaftr control". @@ -194,10 +194,10 @@ A bug-fix and documentation release. * Added performance analysis of the overhead of ingress and egress filtering. See - https://github.com/Igalia/snabb/blob/lwaftr_starfruit/src/program/lwaftr/doc/README.filters-performance.md. + https://github.com/Igalia/snabb/blob/lwaftr_starfruit/src/program/lwaftr/doc/filters-performance.md. * Updated documentation for performance tuning. See - https://github.com/Igalia/snabb/blob/lwaftr_starfruit/src/program/lwaftr/doc/README.performance.md + https://github.com/Igalia/snabb/blob/lwaftr_starfruit/src/program/lwaftr/doc/performance.md * Add a time-stamp for the JIT self-healing behavior, and adapt the message to be more helpful. @@ -253,7 +253,7 @@ A bug fix release. * Add ability to read in ingress and egress filters from files. If the filter value starts with a "<", it is interpreted as a file that should be read. For example, `ipv6_egress_filter = - Date: Fri, 9 Dec 2016 14:58:55 +0100 Subject: [PATCH 432/631] Leader-follower protocol tweaks This change makes the follower apply some pacing so that it doesn't stall packet processing. Also, the follower now waits for a "commit" action to actually apply its changes. The code that creates actions has been changed to add "commit" actions where appropriate. --- src/apps/config/action_codec.lua | 6 ++- src/apps/config/follower.lua | 40 ++++++++++++++----- src/apps/config/support.lua | 1 + src/apps/config/support/snabb-softwire-v1.lua | 3 +- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/apps/config/action_codec.lua b/src/apps/config/action_codec.lua index cacccdad90..fe67c45094 100644 --- a/src/apps/config/action_codec.lua +++ b/src/apps/config/action_codec.lua @@ -12,7 +12,7 @@ local shm = require("core.shm") local action_names = { 'unlink_output', 'unlink_input', 'free_link', 'new_link', 'link_output', 'link_input', 'stop_app', 'start_app', 'reconfig_app', - 'call_app_method_with_blob' } + 'call_app_method_with_blob', 'commit' } local action_codes = {} for i, name in ipairs(action_names) do action_codes[name] = i end @@ -70,6 +70,9 @@ function actions.call_app_method_with_blob (codec, appname, methodname, blob) local blob = codec:blob(blob) return codec:finish(appname, methodname, blob) end +function actions.commit (codec) + return codec:finish() +end local public_names = {} local function find_public_name(obj) @@ -238,5 +241,6 @@ function selftest () test_action({'start_app', {appname, class, arg}}) test_action({'reconfig_app', {appname, class, arg}}) test_action({'call_app_method_with_blob', {appname, methodname, blob}}) + test_action({'commit', {}}) print('selftest: ok') end diff --git a/src/apps/config/follower.lua b/src/apps/config/follower.lua index 83240f0836..39f1a5db40 100644 --- a/src/apps/config/follower.lua +++ b/src/apps/config/follower.lua @@ -23,32 +23,50 @@ function Follower:new (conf) ret.period = 1/conf.Hz ret.next_time = app.now() ret.channel = channel.create('config-follower-channel', 1e6) + ret.pending_actions = {} return ret end -function Follower:handle_actions_from_leader() - local channel = self.channel +function Follower:commit_pending_actions() + local to_apply = {} local should_flush = false - while true do - local buf, len = channel:peek_message() - if not buf then break end - local action = action_codec.decode(buf, len) + for _,action in ipairs(self.pending_actions) do local name, args = unpack(action) if name == 'call_app_method_with_blob' then + if #to_apply > 0 then + app.apply_config_actions(to_apply) + to_apply = {} + end local callee, method, blob = unpack(args) local obj = assert(app.app_table[callee]) assert(obj[method])(obj, blob) else - app.apply_config_actions({action}) - end - channel:discard_message(len) - if action[1] == 'start_app' or action[1] == 'reconfig_app' then - should_flush = true + if name == 'start_app' or name == 'reconfig_app' then + should_flush = true + end + table.insert(to_apply, action) end end + if #to_apply > 0 then app.apply_config_actions(to_apply) end + self.pending_actions = {} if should_flush then require('jit').flush() end end +function Follower:handle_actions_from_leader() + local channel = self.channel + for i=1,10 do + local buf, len = channel:peek_message() + if not buf then break end + local action = action_codec.decode(buf, len) + if action[1] == 'commit' then + self:commit_pending_actions() + else + table.insert(self.pending_actions, action) + end + channel:discard_message(len) + end +end + function Follower:pull () if app.now() < self.next_time then return end self.next_time = app.now() + self.period diff --git a/src/apps/config/support.lua b/src/apps/config/support.lua index c93776818c..d2ecce52f1 100644 --- a/src/apps/config/support.lua +++ b/src/apps/config/support.lua @@ -171,6 +171,7 @@ local function add_restarts(actions, app_graph, to_restart) table.insert(actions, {'link_input', {ta, tl, linkspec}}) end end + table.insert(actions, {'commit', {}}) return actions end diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 3428b07160..656f18bb4b 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -22,6 +22,7 @@ local function add_softwire_entry_actions(app_graph, entries) local args = {'lwaftr', 'add_softwire_entry', blob} table.insert(ret, {'call_app_method_with_blob', args}) end + table.insert(ret, {'commit', {}}) return ret end @@ -44,7 +45,7 @@ local function remove_softwire_entry_actions(app_graph, path) local key = path_mod.prepare_table_lookup( grammar.keys, grammar.key_ctype, path[#path].query) local args = {'lwaftr', 'remove_softwire_entry', key} - return {{'call_app_method_with_blob', args}} + return {{'call_app_method_with_blob', args}, {'commit', {}}} end local function compute_config_actions(old_graph, new_graph, to_restart, From 3066149095d636fb0a25d6f77761fa9f049a3ebf Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 15:14:27 +0000 Subject: [PATCH 433/631] Cache ctable entry types. Before, each ctable creation would result in a fresh struct allocation. This would eventually cause LuaJIT's 65536-entry ctype table to overflow. --- src/lib/ctable.lua | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/ctable.lua b/src/lib/ctable.lua index 294dbdc1fc..35d52044bd 100644 --- a/src/lib/ctable.lua +++ b/src/lib/ctable.lua @@ -19,14 +19,24 @@ local uint16_ptr_t = ffi.typeof('uint16_t*') local uint32_ptr_t = ffi.typeof('uint32_t*') local uint64_ptr_t = ffi.typeof('uint64_t*') +local entry_types = {} local function make_entry_type(key_type, value_type) - return ffi.typeof([[struct { + local cache = entry_types[key_type] + if cache then + cache = cache[value_type] + if cache then return cache end + else + entry_types[key_type] = {} + end + local ret = ffi.typeof([[struct { uint32_t hash; $ key; $ value; } __attribute__((packed))]], key_type, value_type) + entry_types[key_type][value_type] = ret + return ret end local function make_entries_type(entry_type) From d182dc5bcc602566a52310b0c658c3647539e38c Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 15:15:45 +0000 Subject: [PATCH 434/631] Make follower not so thirsty This lowers the effective rate that the follower processes messages to Hz*4, which is really Hz*2 if you consider commit messages. The goal was to reduce packet drop. --- src/apps/config/follower.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/config/follower.lua b/src/apps/config/follower.lua index 39f1a5db40..a50416c77a 100644 --- a/src/apps/config/follower.lua +++ b/src/apps/config/follower.lua @@ -54,7 +54,7 @@ end function Follower:handle_actions_from_leader() local channel = self.channel - for i=1,10 do + for i=1,4 do local buf, len = channel:peek_message() if not buf then break end local action = action_codec.decode(buf, len) From 951fff84000221a71ffde679e305761faef9b4c2 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 9 Dec 2016 14:38:20 +0000 Subject: [PATCH 435/631] Show whether a PID is a worker in 'snabb ps' --- src/program/ps/ps.lua | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index eede9f3f8e..9dd0147f97 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -7,6 +7,8 @@ local lib = require("core.lib") local shm = require("core.shm") local app = require("core.app") +local basename, dirname = lib.basename, lib.dirname + local function usage (code) local f = code == 0 and io.stdout or io.stderr f:write(require("program.ps.README_inc")) @@ -31,6 +33,16 @@ local function appname_resolver() return function (pid) return instances[pid] end end +local function is_worker (pid) + return shm.exists("/"..pid.."/group") +end + +local function get_leader_pid (pid) + local fq = shm.root.."/"..pid.."/group" + local path = S.readlink(fq) + return basename(dirname(path)) +end + local function compute_snabb_instances() -- Produces set of snabb instances, excluding this one. local whichname = appname_resolver() @@ -40,7 +52,13 @@ local function compute_snabb_instances() -- This could fail as the name could be for example "by-name" local p = tonumber(name) local name = whichname(p) - if p and p ~= my_pid then table.insert(pids, {pid=p, name=name}) end + if p and p ~= my_pid then + local instance = {pid=p, name=name} + if is_worker(p) then + instance.leader = get_leader_pid(p) + end + table.insert(pids, instance) + end end return pids end @@ -53,10 +71,15 @@ function run (args) print(instance.pid) else if instance.name then - print(instance.name) + io.write(instance.name) else - print(instance.pid) + io.write(instance.pid) + end + -- Check instance is a worker. + if instance.leader then + io.write(("\tWorker for PID %s"):format(instance.leader)) end + io.write("\n") end end main.exit(0) From 98216ef96db070a11d7388dd9133ba6067378539 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 9 Dec 2016 14:49:55 +0000 Subject: [PATCH 436/631] Sort list of pids --- src/program/ps/ps.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index 9dd0147f97..a46ba428cc 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -60,6 +60,9 @@ local function compute_snabb_instances() table.insert(pids, instance) end end + table.sort(pids, function(a, b) + return tonumber(a.pid) < tonumber(b.pid) + end) return pids end From abb93723f9eb1e68932d021a1fdda00e73357c8b Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 9 Dec 2016 15:22:16 +0000 Subject: [PATCH 437/631] Show PIDs as a tree Print out PID name, if defined, on a second column. --- src/program/ps/README | 9 +++------ src/program/ps/ps.lua | 21 +++++++-------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/program/ps/README b/src/program/ps/README index ef1db90734..2bdea1ae30 100644 --- a/src/program/ps/README +++ b/src/program/ps/README @@ -3,11 +3,8 @@ Print a list of running Snabb instances. Available options: -h, --help Display this message. - -p, --PID Display processes by their PID instead of name. -The default output format is one identifier by line. If the --p or --pid flag is -used then the it will ignore any name defined and list just the PID. More -options can be added in the future. +The default output format is one identifier by line. -The identifier is either the name (preferentially) or, if one is not defined, -the PID of the process is used. +An identifier is a Snabb instance PID. If a name is defined for the PID, +its value is printed on a second column. diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index a46ba428cc..8fc2cd2aee 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -17,12 +17,9 @@ end local function parse_args (args) local opt = {} - local preferpid = false function opt.h (arg) usage(0) end - function opt.p (arg) preferpid = true end - args = lib.dogetopt(args, opt, "hp", {help='h', pid='p'}) + args = lib.dogetopt(args, opt, "h", {help='h'}) if #args ~= 0 then usage(1) end - return preferpid end local function appname_resolver() @@ -67,20 +64,16 @@ local function compute_snabb_instances() end function run (args) - local preferpid = parse_args (args) + parse_args(args) local instances = compute_snabb_instances() for _, instance in ipairs(instances) do - if preferpid then - print(instance.pid) + -- Check instance is a worker. + if instance.leader then + print(" \\- "..instance.pid.." worker for "..instance.leader) else + io.write(instance.pid) if instance.name then - io.write(instance.name) - else - io.write(instance.pid) - end - -- Check instance is a worker. - if instance.leader then - io.write(("\tWorker for PID %s"):format(instance.leader)) + io.write("\t["..instance.name.."]") end io.write("\n") end From e0c039d587530d2f175d23a46fea638f224562f1 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 16:31:54 +0000 Subject: [PATCH 438/631] Close pipes before execcing listen --- src/program/config/bench/bench.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/program/config/bench/bench.lua b/src/program/config/bench/bench.lua index 283c2a20ab..f08a1f153a 100644 --- a/src/program/config/bench/bench.lua +++ b/src/program/config/bench/bench.lua @@ -93,6 +93,8 @@ function run(args) output_read:close() assert(S.dup2(input_read, 0)) assert(S.dup2(output_write, 1)) + input_read:close() + output_write:close() S.execve(("/proc/%d/exe"):format(S.getpid()), argv, {}) end input_read:close() From 4fd4aa77a89b8dd1074962413e59dd11081ec7cb Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 17:33:44 +0100 Subject: [PATCH 439/631] Sketch out Igalia's testing strategy --- src/program/lwaftr/doc/testing.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/program/lwaftr/doc/testing.md diff --git a/src/program/lwaftr/doc/testing.md b/src/program/lwaftr/doc/testing.md new file mode 100644 index 0000000000..f16c4f80e5 --- /dev/null +++ b/src/program/lwaftr/doc/testing.md @@ -0,0 +1,13 @@ +# Testing + +The lwAFTR and associated software is tested in several ways. Igalia uses unit tests, integration tests ("selftest.sh"), end-to-end tests, and does an increasing amount of automated performance analysis with Hydra. + +An example of our testing is how ARP support is tested. ARP is implemented in a small app, independently of the lwAFTR, and included with the lwAFTR an app network within Snabb. It has unit tests (in the function 'selftest()', following the convention to run unit tests automatically), which check how the request is formed, and the reply the library gives to that request. We also have two end-to-end tests for ARP; these specify an incoming packet, pass it through the whole app network, and make sure that the end result is byte for byte identical with what is expected. Both the implementation and the testing were done with careful attention paid to RFC 826 throughout, as well as to dumps of ARP packets from the live network the developer was on. + +There are a hierarchy of tests. Unit tests are internally orientedf, and make sure that basic functionality and some edge cases are tested. By convention, they are found in functions called selftest(). + +Integration tests tend to be called selftest.sh, or invoked by selftesh.sh. These test larger components of the system. The end to end tests have IPv4 and IPv6 packet captures, run them through the lwaftr app network, and compare the results with predetermined captures and counter values, making sure everything works as expected. These test a large variety of RFC-mandated behaviours. + +We also have some 'soak tests', which repeatedly do the same thing a large number of times; these show that the system holds up under the tested heavy workloads, without errors like memory corruption or memory leaks, which would cause them to fail. + +Igalia is following the lead of upstream Snabb in automated performance testing. This is a work in progress, and can be seen at https://hydra.snabb.co/project/igalia . We are also developing property-based tests to stress-test our yang infrastructure, and have written a load tester for our yang implementation as well, verifying that it is performant. From c1008a65c66c378042cb58a5865f5a7986354785 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Fri, 9 Dec 2016 17:35:50 +0100 Subject: [PATCH 440/631] Remove useless pcap files (#667) Remove useless pcap files, clarify docs and code. --- src/program/lwaftr/doc/running.md | 12 +++++------- src/program/lwaftr/loadtest/loadtest.lua | 5 ++++- .../lwaftr/tests/benchdata/ipv4-imix.pcap | Bin 3896 -> 0 bytes .../lwaftr/tests/benchdata/ipv6-imix.pcap | Bin 4256 -> 0 bytes 4 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 src/program/lwaftr/tests/benchdata/ipv4-imix.pcap delete mode 100644 src/program/lwaftr/tests/benchdata/ipv6-imix.pcap diff --git a/src/program/lwaftr/doc/running.md b/src/program/lwaftr/doc/running.md index 806e71db6d..e9e91ab4c3 100644 --- a/src/program/lwaftr/doc/running.md +++ b/src/program/lwaftr/doc/running.md @@ -74,18 +74,16 @@ $ sudo ./snabb lwaftr run \ --on-a-stick 0000:02:00.1 ``` -You can run a load generator: +You can run a load generator in on-a-stick mode: ``` $ sudo ./snabb lwaftr loadtest \ - program/lwaftr/tests/benchdata/ipv4-0550.pcap IPv4 IPv6 0000:82:00.0 \ - program/lwaftr/tests/benchdata/ipv6-0550.pcap IPv6 IPv4 0000:82:00.1 + program/lwaftr/tests/benchdata/ipv4_and_ipv6_stick_imix.pcap ALL ALL \ + 0000:82:00.1 ``` -For the main lwaftr instance to receive any traffic, either 82:00.0 or 82:00.1 -should be wired to 02:00.1 . Since `lwaftr loadtest` does not currently have -an on-a-stick mode, the main lwaftr instance will only see half the loadtest -traffic. +For the main lwaftr instance to receive any traffic, the 82:00.1 port should be +wired to the 02:00.1 one. ## The packetblaster diff --git a/src/program/lwaftr/loadtest/loadtest.lua b/src/program/lwaftr/loadtest/loadtest.lua index 1ac5597efd..50f36d35d2 100644 --- a/src/program/lwaftr/loadtest/loadtest.lua +++ b/src/program/lwaftr/loadtest/loadtest.lua @@ -240,8 +240,11 @@ function run(args) print(string.format(' Loss: %d ingress drop + %d packets lost (%f%%)', drop, lost_packets, lost_percent)) if hydra_mode then - -- Hydra reports prefer integers for the X (time) axis. + -- NOTE: all the stats below are available: the commented out ones + -- will not show in Hydra reports. They are too many, making the + -- graphs unreadable, and most are redundant anyway. -- TX + -- (Hydra reports prefer integers for the X (time) axis.) -- bench_file:write(('%s_tx_packets,%.f,%f,packets\n'):format( -- stream.tx_name,gbps_bitrate,tx.txpackets)) -- bench_file:write(('%s_tx_mpps,%.f,%f,mpps\n'):format( diff --git a/src/program/lwaftr/tests/benchdata/ipv4-imix.pcap b/src/program/lwaftr/tests/benchdata/ipv4-imix.pcap deleted file mode 100644 index beabe457bfe8ae0ac70df65ce117e0e4b4a46106..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3896 zcmca|c+)~A1{MYw`2U}Qff2?5(tc1Z1O^ZZ4hB~S1_uzEUohC00|++xPh&K&1e&74 zz;N!@zx5Dxgh}#DM|K}RJIFJg37?Tja^#uLjAQ~nCV8eK;t`+O3=9cCPl7OdtTF|G z^zjP@S|i6QlNlsdS@9b>DmxkiI6{ED^u~t6v{B*F5WpV-kREp^nF0NT0& z*}wQ)Lt9rc;jK-+Z5hQroT;n5Jl9|E*Zm+bg0AC(;q0URM91T4}(7_%N{y+cer j&Uy!AFnT@CdH_<7bKtObRCqK5MnhmU1V%$(@P+^Y)nI-F From 10e84dde14fd97a81b7c37823dad389dc48c3a4e Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 17:42:52 +0100 Subject: [PATCH 441/631] Hard wrap at 80 --- src/program/lwaftr/doc/testing.md | 36 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/program/lwaftr/doc/testing.md b/src/program/lwaftr/doc/testing.md index f16c4f80e5..a5ed1a4986 100644 --- a/src/program/lwaftr/doc/testing.md +++ b/src/program/lwaftr/doc/testing.md @@ -1,13 +1,37 @@ # Testing -The lwAFTR and associated software is tested in several ways. Igalia uses unit tests, integration tests ("selftest.sh"), end-to-end tests, and does an increasing amount of automated performance analysis with Hydra. +The lwAFTR and associated software is tested in several ways. Igalia uses unit +tests, integration tests ("selftest.sh"), end-to-end tests, and does an +increasing amount of automated performance analysis with Hydra. -An example of our testing is how ARP support is tested. ARP is implemented in a small app, independently of the lwAFTR, and included with the lwAFTR an app network within Snabb. It has unit tests (in the function 'selftest()', following the convention to run unit tests automatically), which check how the request is formed, and the reply the library gives to that request. We also have two end-to-end tests for ARP; these specify an incoming packet, pass it through the whole app network, and make sure that the end result is byte for byte identical with what is expected. Both the implementation and the testing were done with careful attention paid to RFC 826 throughout, as well as to dumps of ARP packets from the live network the developer was on. +An example of our testing is how ARP support is tested. ARP is implemented in a +small app, independently of the lwAFTR, and included with the lwAFTR an app +network within Snabb. It has unit tests (in the function 'selftest()', following +the convention to run unit tests automatically), which check how the request is +formed, and the reply the library gives to that request. We also have two +end-to-end tests for ARP; these specify an incoming packet, pass it through the +whole app network, and make sure that the end result is byte for byte identical +with what is expected. Both the implementation and the testing were done with +careful attention paid to RFC 826 throughout, as well as to dumps of ARP packets +from the live network the developer was on. -There are a hierarchy of tests. Unit tests are internally orientedf, and make sure that basic functionality and some edge cases are tested. By convention, they are found in functions called selftest(). +There are a hierarchy of tests. Unit tests are internally orientedf, and make +sure that basic functionality and some edge cases are tested. By convention, +they are found in functions called selftest(). -Integration tests tend to be called selftest.sh, or invoked by selftesh.sh. These test larger components of the system. The end to end tests have IPv4 and IPv6 packet captures, run them through the lwaftr app network, and compare the results with predetermined captures and counter values, making sure everything works as expected. These test a large variety of RFC-mandated behaviours. +Integration tests tend to be called selftest.sh, or invoked by selftesh.sh. +These test larger components of the system. The end to end tests have IPv4 and +IPv6 packet captures, run them through the lwaftr app network, and compare the +results with predetermined captures and counter values, making sure everything +works as expected. These test a large variety of RFC-mandated behaviours. -We also have some 'soak tests', which repeatedly do the same thing a large number of times; these show that the system holds up under the tested heavy workloads, without errors like memory corruption or memory leaks, which would cause them to fail. +We also have some 'soak tests', which repeatedly do the same thing a large +number of times; these show that the system holds up under the tested heavy +workloads, without errors like memory corruption or memory leaks, which would +cause them to fail. -Igalia is following the lead of upstream Snabb in automated performance testing. This is a work in progress, and can be seen at https://hydra.snabb.co/project/igalia . We are also developing property-based tests to stress-test our yang infrastructure, and have written a load tester for our yang implementation as well, verifying that it is performant. +Igalia is following the lead of upstream Snabb in automated performance testing. +This is a work in progress, and can be seen at +https://hydra.snabb.co/project/igalia . We are also developing property-based +tests to stress-test our yang infrastructure, and have written a load tester for +our yang implementation as well, verifying that it is performant. From 58315b1ce29d8eb1e191134560e55a6affc37e64 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 17:50:47 +0100 Subject: [PATCH 442/631] Explained how to test ARP in the wild --- src/program/lwaftr/doc/testing.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/program/lwaftr/doc/testing.md b/src/program/lwaftr/doc/testing.md index a5ed1a4986..6757f4a175 100644 --- a/src/program/lwaftr/doc/testing.md +++ b/src/program/lwaftr/doc/testing.md @@ -15,6 +15,15 @@ with what is expected. Both the implementation and the testing were done with careful attention paid to RFC 826 throughout, as well as to dumps of ARP packets from the live network the developer was on. +ARP has two main parts: resolving a remote address, and providing the address of +the lwAFTR on request. The first one can be tested in a network by specifying +only the remote IP of the next IPv4 hop (not the ethernet address), then sending +packets through the lwAFTR and confirming on the remote host that they are +arriving. The latter can be tested by issuing an ARP request to the lwaftr from +another machine; if the other machine runs Linux, `arp -n` should then show a +new entry corresponding to the lwAFTR. The end to end tests simulate both of +these cases, but with captured packets rather than a live network. + There are a hierarchy of tests. Unit tests are internally orientedf, and make sure that basic functionality and some edge cases are tested. By convention, they are found in functions called selftest(). From e99ad1a979ecdb6eb07deeb1151e58f1f9386ebe Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 17:56:39 +0100 Subject: [PATCH 443/631] Hard word wrap at 80 chars for filter perf doc --- src/program/lwaftr/doc/filters-performance.md | 74 +++++++++++++------ 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/src/program/lwaftr/doc/filters-performance.md b/src/program/lwaftr/doc/filters-performance.md index 3559cc4f52..715981d015 100644 --- a/src/program/lwaftr/doc/filters-performance.md +++ b/src/program/lwaftr/doc/filters-performance.md @@ -2,26 +2,42 @@ ## Summarized results -Having filters, even if they are empty, has a significant negative impact on performance. +Having filters, even if they are empty, has a significant negative impact on +performance. Here are the results for three runs with empty filters: -* No packet loss below 7 Gbps -* Packet loss at 8 Gbps on cooldown on two of the three runs (4.3% and 3.9%). -* Packet loss at 9 Gbps on all three runs (warmup: 0.13%, 14.6%, 14.2%; cooldowns marginally worse) -* Heavy packet loss at 10 Gbps: (10.4-10.6%, 23.2-23.6%, 22.9-23.2%) +* No packet loss below 7 Gbps Packet loss at 8 Gbps on cooldown on two of the +* three runs (4.3% and 3.9%). Packet loss at 9 Gbps on all three runs (warmup: +* 0.13%, 14.6%, 14.2%; cooldowns marginally worse) Heavy packet loss at 10 Gbps: +* (10.4-10.6%, 23.2-23.6%, 22.9-23.2%) -Results appear to be approximately the same with one filter. Scaling it up to 800 filters, results are a bit worse; packet loss starts at 7 Gbps, and peak packet loss at 10 Gbps is around 34%. +Results appear to be approximately the same with one filter. Scaling it up to +800 filters, results are a bit worse; packet loss starts at 7 Gbps, and peak +packet loss at 10 Gbps is around 34%. -The load that was applied was 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 Gbps, on each of two interfaces at the same time; the total traffic going into the lwaftr was twice as high, due to their being two (equally loaded) interfaces. The latter half of the load, after it peaks at 10 Gbps per card, is referred to as "cooldown". +The load that was applied was 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 9, 8, 7, 6, 5, +4, 3, 2, 1 Gbps, on each of two interfaces at the same time; the total traffic +going into the lwaftr was twice as high, due to their being two (equally loaded) +interfaces. The latter half of the load, after it peaks at 10 Gbps per card, is +referred to as "cooldown". ## Future improvements -The nature of packet loss with empty filters suggests that the overhead of adding four extra apps to the Snabb app network (one per filter option in the config file) is the critical problem. We could integrate the filters into the lwaftr app itself to side-step this. Alternatively, on an "on-a-stick" configuration that only uses one rather than two cards, the overhead might be small enough to still not matter, even with 800 filters; there was no packet loss (except on cooldown) even with 800 filters at 5 Gbps or 6 Gbps per interface with two interfaces. +The nature of packet loss with empty filters suggests that the overhead of +adding four extra apps to the Snabb app network (one per filter option in the +config file) is the critical problem. We could integrate the filters into the +lwaftr app itself to side-step this. Alternatively, on an "on-a-stick" +configuration that only uses one rather than two cards, the overhead might be +small enough to still not matter, even with 800 filters; there was no packet +loss (except on cooldown) even with 800 filters at 5 Gbps or 6 Gbps per +interface with two interfaces. # Details of the results and configuration -Setup: bidirectional lwaftr on snabb1, load generation on snabb2. Using Snabb revision 79504183e1acb5673f7eda9d788885ff8c076f39 (lwaftr_starfruit branch, Igalia fork) +Setup: bidirectional lwaftr on snabb1, load generation on snabb2. Using Snabb +revision 79504183e1acb5673f7eda9d788885ff8c076f39 (lwaftr_starfruit branch, +Igalia fork) ## Running the lwaftr (with taskset and numactl) @@ -31,7 +47,8 @@ $ cat ~/bin/run-lwaftr #!/bin/sh BASEDIR=/home/kbarone/snabbswitch/src/ CONF="`realpath $1`" -cd ${BASEDIR} && sudo numactl -m 0 taskset -c 1 ./snabb lwaftr run --conf ${CONF} --v4-pci 0000:02:00.0 --v6-pci 0000:02:00.1 +cd ${BASEDIR} && sudo numactl -m 0 taskset -c 1 \ + ./snabb lwaftr run --conf ${CONF} --v4-pci 0000:02:00.0 --v6-pci 0000:02:00.1 ``` ## Load generation @@ -85,9 +102,12 @@ softwires { ## Baseline (no filters), _run-lwaftr no_filters.conf_ -There is only loss at 10 Gbps, and it is only what is logically expected when packets from a saturated link are made larger; with 550 byte packets, it is 6.5%. +There is only loss at 10 Gbps, and it is only what is logically expected when +packets from a saturated link are made larger; with 550 byte packets, it is +6.5%. -This was run three times to verify consistency. All the results were essentially the same; a snippet of the first is below. +This was run three times to verify consistency. All the results were essentially +the same; a snippet of the first is below. ``` Applying 9.000000 Gbps of load. @@ -141,12 +161,13 @@ ipv6_egress_filter = "" , ### Results with empty filters -* No packet loss below 7 Gbps -* Packet loss at 8 Gbps on cooldown on two of the three runs (4.3% and 3.9%). -* Packet loss at 9 Gbps on all three runs (warmup: 0.13%, 14.6%, 14.2%; cooldowns marginally worse) -* Heavy packet loss at 10 Gbps: (10.4-10.6%, 23.2-23.6%, 22.9-23.2%) +* No packet loss below 7 Gbps Packet loss at 8 Gbps on cooldown on two of the +* three runs (4.3% and 3.9%). Packet loss at 9 Gbps on all three runs (warmup: +* 0.13%, 14.6%, 14.2%; cooldowns marginally worse) Heavy packet loss at 10 +* Gbps: (10.4-10.6%, 23.2-23.6%, 22.9-23.2%) -Results tentatively appear similar whether the empty filters are specified directly or in a file. +Results tentatively appear similar whether the empty filters are specified +directly or in a file. ### Empty filters, Run 1 @@ -424,10 +445,13 @@ Applying 7.000000 Gbps of load. ### One filter per option (4 total), _run-lwaftr single_filters.conf_ -The results on one run were similar to with empty filters - actually, slightly better than the best empty filter run, though that is almost certainly noise. +The results on one run were similar to with empty filters - actually, slightly +better than the best empty filter run, though that is almost certainly noise. -Summary: ramp-up 9 Gbps had 0.6% packet loss, 10 Gpbs had 9.4-9.5% packet loss, cooldown 9 Gbps had 0.03% packet loss, cooldown 8 Gbps had 0.2% packet loss. -That is not a typo; there was ~10 times as much packet loss at 8 Gbps than at 9 Gbps on the cooldown. +Summary: ramp-up 9 Gbps had 0.6% packet loss, 10 Gpbs had 9.4-9.5% packet loss, +cooldown 9 Gbps had 0.03% packet loss, cooldown 8 Gbps had 0.2% packet loss. +That is not a typo; there was ~10 times as much packet loss at 8 Gbps than at 9 +Gbps on the cooldown. Changes to the configuration file: ``` @@ -510,8 +534,10 @@ This has 200 filters per ingress/egress option, or 800 total. The filters were generated with the following bash script: ``` -for x in {0..99}; do echo "not src host 192.168.255.${x}" >> 00200v4.pf; echo "not ether host 1:2:3:4:5:${x}" >> 00200v4.pf; done -for x in {0..99}; do echo "not src host 1::${x}" >> 00200v6.pf; echo "not ether host 1:2:3:4:5:${x}" >> 00200v6.pf; done +for x in {0..99}; do echo "not src host 192.168.255.${x}" >> 00200v4.pf; +echo "not ether host 1:2:3:4:5:${x}" >> 00200v4.pf; done +for x in {0..99}; do echo "not src host 1::${x}" >> 00200v6.pf; +echo "not ether host 1:2:3:4:5:${x}" >> 00200v6.pf; done ``` The configuration file changes: @@ -522,7 +548,9 @@ ipv6_ingress_filter = <00200v6.pf, ipv6_egress_filter = <00200v6.pf, ``` -Results: packet loss starts at 7 Gbps, peaking around 33-34% at 10 Gbps, and reaching 0 again at 4-6 Gbps during the cooldown, depending on the run. Note that this is made noisier by the jit.flush() overhead. +Results: packet loss starts at 7 Gbps, peaking around 33-34% at 10 Gbps, and +reaching 0 again at 4-6 Gbps during the cooldown, depending on the run. Note +that this is made noisier by the jit.flush() overhead. ### Run 1, 800 filters From 2c16488e493c7a8309f24d832fa2e9665405569b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 17:09:09 +0000 Subject: [PATCH 444/631] Fix snabb config listen for get-state. --- src/program/config/listen/listen.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 28739af12a..47d29b007c 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -51,8 +51,11 @@ local function read_request(client, schema_name, revision_date) local req = handler(schema_name, revision_date, path, json.value) local function print_reply(reply) local output = json_lib.buffered_output() - json_lib.write_json_object(output, - {id=id, status='ok', value=reply.config}) + local value + if verb == 'get' then value = reply.config + elseif verb == 'get-state' then value = reply.state + end + json_lib.write_json_object(output, {id=id, status='ok', value=value}) output:flush(S.stdout) end return req, print_reply From 3eaf558b3bcc2e528a843744b24d0e989ad5c23a Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 18:11:23 +0100 Subject: [PATCH 445/631] Fixed typos, explained modular philosophy --- src/program/lwaftr/doc/testing.md | 32 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/program/lwaftr/doc/testing.md b/src/program/lwaftr/doc/testing.md index 6757f4a175..9867afd8f8 100644 --- a/src/program/lwaftr/doc/testing.md +++ b/src/program/lwaftr/doc/testing.md @@ -4,33 +4,39 @@ The lwAFTR and associated software is tested in several ways. Igalia uses unit tests, integration tests ("selftest.sh"), end-to-end tests, and does an increasing amount of automated performance analysis with Hydra. -An example of our testing is how ARP support is tested. ARP is implemented in a -small app, independently of the lwAFTR, and included with the lwAFTR an app -network within Snabb. It has unit tests (in the function 'selftest()', following -the convention to run unit tests automatically), which check how the request is -formed, and the reply the library gives to that request. We also have two -end-to-end tests for ARP; these specify an incoming packet, pass it through the -whole app network, and make sure that the end result is byte for byte identical -with what is expected. Both the implementation and the testing were done with -careful attention paid to RFC 826 throughout, as well as to dumps of ARP packets -from the live network the developer was on. +The overall project is developed as a series of smaller, more modular parts. +For example, facilities like ARP and ping support are implemented as Snabb +apps, independently of the main lwAFTR app; they are then all combined into an +app network within Snabb. YANG support has a similar flavour; the majority of +it is run in a separate process from the data plane. This makes it easier to +test, easier to extend and change, and more reliable. +An example of our testing is how ARP support is tested. It has unit tests (in +the function 'selftest()', following the convention to run unit tests +automatically), which check how the request is formed, and the reply the library +gives to that request. We also have two end-to-end tests for ARP; these specify +an incoming packet, pass it through the whole app network, and make sure that +the end result is byte for byte identical with what is expected. Both the +implementation and the testing were done with careful attention paid to RFC 826 +throughout, as well as to dumps of ARP packets from the live network the +developer was on. + ARP has two main parts: resolving a remote address, and providing the address of the lwAFTR on request. The first one can be tested in a network by specifying only the remote IP of the next IPv4 hop (not the ethernet address), then sending packets through the lwAFTR and confirming on the remote host that they are -arriving. The latter can be tested by issuing an ARP request to the lwaftr from +arriving. The latter can be tested by issuing an ARP request to the lwAFTR from another machine; if the other machine runs Linux, `arp -n` should then show a new entry corresponding to the lwAFTR. The end to end tests simulate both of these cases, but with captured packets rather than a live network. -There are a hierarchy of tests. Unit tests are internally orientedf, and make +There are a hierarchy of tests. Unit tests are internally oriented, and make sure that basic functionality and some edge cases are tested. By convention, they are found in functions called selftest(). Integration tests tend to be called selftest.sh, or invoked by selftesh.sh. These test larger components of the system. The end to end tests have IPv4 and -IPv6 packet captures, run them through the lwaftr app network, and compare the +IPv6 packet captures, run them through the lwAFTR app network, and compare the results with predetermined captures and counter values, making sure everything works as expected. These test a large variety of RFC-mandated behaviours. From def9ad0fea7739a6122ece877bc19a5307265cea Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 9 Dec 2016 17:11:54 +0000 Subject: [PATCH 446/631] Mark processes that are addressable by snabb config --- src/program/ps/ps.lua | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index 8fc2cd2aee..0d4bfaa5d4 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -34,6 +34,18 @@ local function is_worker (pid) return shm.exists("/"..pid.."/group") end +local function is_addressable (pid) + local socket = assert(S.socket("unix", "stream")) + local tail = pid.."/config-leader-socket" + local by_name = S.t.sockaddr_un(shm.root..'/by-name/'..tail) + local by_pid = S.t.sockaddr_un(shm.root..'/'..tail) + if socket:connect(by_name) or socket:connect(by_pid) then + socket:close() + return true + end + return false +end + local function get_leader_pid (pid) local fq = shm.root.."/"..pid.."/group" local path = S.readlink(fq) @@ -54,6 +66,9 @@ local function compute_snabb_instances() if is_worker(p) then instance.leader = get_leader_pid(p) end + if is_addressable(p) then + instance.addressable = true + end table.insert(pids, instance) end end @@ -69,14 +84,17 @@ function run (args) for _, instance in ipairs(instances) do -- Check instance is a worker. if instance.leader then - print(" \\- "..instance.pid.." worker for "..instance.leader) + io.write(" \\- "..instance.pid.." worker for "..instance.leader) else io.write(instance.pid) if instance.name then io.write("\t["..instance.name.."]") end - io.write("\n") end + if instance.addressable then + io.write(" *") + end + io.write("\n") end main.exit(0) end From df3484c5f0e281e70787825838d36ee49326bf8a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 18:27:02 +0100 Subject: [PATCH 447/631] Snabb lwAFTR v3.1.5 change log --- src/program/lwaftr/doc/CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index e6c1f6ae15..58eb47d048 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,18 @@ # Change Log +## [3.1.5] - 2016-12-09 + + * Fix race condition in multiprocess --reconfigurable mode. + + * Improve configuration change throughput. + + * Add "snabb config bench" utility for benchmarking configuration + throughput. + + * Add automated "snabb config" tests. + + * Improve error message when --cpu setting was not possible. + ## [3.1.4] - 2016-12-09 * Fix memory corruption bug in main process of --reconfigurable "snabb From 0183f4333639820d0c280a4349c99e3550827ce2 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 9 Dec 2016 18:29:34 +0100 Subject: [PATCH 448/631] Snabb lwAFTR v3.1.5 change log --- src/program/lwaftr/doc/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 58eb47d048..6b1f7d75ca 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -2,6 +2,9 @@ ## [3.1.5] - 2016-12-09 + * Improve "snabb ps" output. Processes with a "*" by them are + listening for "snabb config" connections. + * Fix race condition in multiprocess --reconfigurable mode. * Improve configuration change throughput. From 12297060517706c49ea22495044be16ef44a4866 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 20 Dec 2016 12:50:10 +0100 Subject: [PATCH 449/631] Add basic error reporting to snabb-softwire-v1 --- src/apps/config/leader.lua | 101 +++++++++++++-------- src/lib/yang/snabb-config-leader-v1.yang | 23 +++++ src/program/config/add/add.lua | 2 +- src/program/config/common.lua | 9 ++ src/program/config/get/get.lua | 3 +- src/program/config/get_state/get_state.lua | 9 +- src/program/config/load/load.lua | 2 +- src/program/config/remove/remove.lua | 2 +- src/program/config/set/set.lua | 2 +- 9 files changed, 103 insertions(+), 50 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index cc5d58b18b..d2e871bb47 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -131,12 +131,16 @@ local function path_printer_for_schema_by_name(schema_name, path) end function Leader:rpc_get_config (args) - if args.schema ~= self.schema_name then - return self:foreign_rpc_get_config(args.schema, args.path) + local function getter() + if args.schema ~= self.schema_name then + return self:foreign_rpc_get_config(args.schema, args.path) + end + local printer = path_printer_for_schema_by_name(args.schema, args.path) + local config = printer(self.current_configuration, yang.string_output_file()) + return { config = config } end - local printer = path_printer_for_schema_by_name(args.schema, args.path) - local config = printer(self.current_configuration, yang.string_output_file()) - return { config = config } + local success, response = pcall(getter) + if success then return response else return {status=1, error=response} end end local function path_parser_for_grammar(grammar, path) @@ -476,58 +480,77 @@ function Leader:foreign_rpc_remove_config (schema_name, path) end function Leader:rpc_set_config (args) - if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then - error('Attempt to modify configuration while listener attached') - end - if args.schema ~= self.schema_name then - return self:foreign_rpc_set_config(args.schema, args.path, args.config) + local function setter() + if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then + error('Attempt to modify configuration while listener attached') + end + if args.schema ~= self.schema_name then + return self:foreign_rpc_set_config(args.schema, args.path, args.config) + end + return self:handle_rpc_update_config(args, 'set', compute_set_config_fn) end - return self:handle_rpc_update_config(args, 'set', compute_set_config_fn) + local success, response = pcall(setter) + if success then return response else return {status=1, error=response} end end function Leader:rpc_add_config (args) - if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then - error('Attempt to modify configuration while listener attached') - end - if args.schema ~= self.schema_name then - return self:foreign_rpc_add_config(args.schema, args.path, args.config) + local function adder() + if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then + error('Attempt to modify configuration while listener attached') + end + if args.schema ~= self.schema_name then + return self:foreign_rpc_add_config(args.schema, args.path, args.config) + end + return self:handle_rpc_update_config(args, 'add', compute_add_config_fn) end - return self:handle_rpc_update_config(args, 'add', compute_add_config_fn) + local success, response = pcall(adder) + if success then return response else return {status=1, error=response} end end function Leader:rpc_remove_config (args) - if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then - error('Attempt to modify configuration while listener attached') - end - if args.schema ~= self.schema_name then - return self:foreign_rpc_remove_config(args.schema, args.path) + local function remover() + if self.listen_peer ~= nil and self.listen_peer ~= self.rpc_peer then + error('Attempt to modify configuration while listener attached') + end + if args.schema ~= self.schema_name then + return self:foreign_rpc_remove_config(args.schema, args.path) + end + local path = path_mod.normalize_path(args.path) + self:update_configuration(compute_remove_config_fn(args.schema, path), + 'remove', path) + return {} end - local path = path_mod.normalize_path(args.path) - self:update_configuration(compute_remove_config_fn(args.schema, path), - 'remove', path) - return {} + local success, response = pcall(remover) + if success then return response else return {status=1, error=response} end end function Leader:rpc_attach_listener (args) - if self.listen_peer ~= nil then error('Listener already attached') end - self.listen_peer = self.rpc_peer - return {} + local function attacher() + if self.listen_peer ~= nil then error('Listener already attached') end + self.listen_peer = self.rpc_peer + return {} + end + local success, response = pcall(attacher) + if success then return response else return {status=1, error=response} end end function Leader:rpc_get_state (args) - if args.schema ~= self.schema_name then - return self:foreign_rpc_get_state(args.schema, args.path) - end - local printer = path_printer_for_schema_by_name(self.schema_name, args.path) - local s = {} - for _, follower in pairs(self.followers) do - for k,v in pairs(state.show_state(self.schema_name, follower.pid, args.path)) do - s[k] = v + local function getter() + if args.schema ~= self.schema_name then + return self:foreign_rpc_get_state(args.schema, args.path) + end + local printer = path_printer_for_schema_by_name(self.schema_name, args.path) + local s = {} + for _, follower in pairs(self.followers) do + for k,v in pairs(state.show_state(self.schema_name, follower.pid, args.path)) do + s[k] = v + end end + return {state=printer(s, yang.string_output_file())} end - return {state=printer(s, yang.string_output_file())} + local success, response = pcall(getter) + if success then return response else return {status=1, error=response} end end - function Leader:handle (payload) return rpc.handle_calls(self.rpc_callee, payload, self.rpc_handler) end diff --git a/src/lib/yang/snabb-config-leader-v1.yang b/src/lib/yang/snabb-config-leader-v1.yang index 0b6b4a3a31..4ec5066826 100644 --- a/src/lib/yang/snabb-config-leader-v1.yang +++ b/src/lib/yang/snabb-config-leader-v1.yang @@ -7,11 +7,20 @@ module snabb-config-leader-v1 { description "RPC interface for ConfigLeader Snabb app."; + revision 2016-12-20 { + description "Add basic error reporting."; + } + revision 2016-11-12 { description "Initial revision."; } + grouping error-reporting { + leaf status { type uint8; default 0; } + leaf error { type string; } + } + rpc describe { output { leaf native-schema { type string; mandatory true; } @@ -31,6 +40,7 @@ module snabb-config-leader-v1 { leaf path { type string; default "/"; } } output { + uses error-reporting; leaf config { type string; } } } @@ -42,6 +52,9 @@ module snabb-config-leader-v1 { leaf path { type string; default "/"; } leaf config { type string; mandatory true; } } + output { + uses error-reporting; + } } rpc add-config { @@ -51,6 +64,9 @@ module snabb-config-leader-v1 { leaf path { type string; mandatory true; } leaf config { type string; mandatory true; } } + output { + uses error-reporting; + } } rpc remove-config { @@ -59,6 +75,9 @@ module snabb-config-leader-v1 { leaf revision { type string; } leaf path { type string; mandatory true; } } + output { + uses error-reporting; + } } rpc get-state { @@ -68,6 +87,7 @@ module snabb-config-leader-v1 { leaf path { type string; default "/"; } } output { + uses error-reporting; leaf state { type string; } } } @@ -77,5 +97,8 @@ module snabb-config-leader-v1 { leaf schema { type string; mandatory true; } leaf revision { type string; } } + output { + uses error-reporting; + } } } diff --git a/src/program/config/add/add.lua b/src/program/config/add/add.lua index dcfcd6fe21..2c50f5463a 100644 --- a/src/program/config/add/add.lua +++ b/src/program/config/add/add.lua @@ -13,5 +13,5 @@ function run(args) config = common.serialize_config( args.value, args.schema_name, args.path) }) -- The reply is empty. - main.exit(0) + common.print_and_exit(response) end diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 30037cec2f..4250053a9f 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -141,3 +141,12 @@ function call_leader(instance_id, method, args) socket:close() return parse_reply(reply) end + +function print_and_exit(response, response_prop) + if response.error then + print(response.error) + elseif response_prop then + print(response[response_prop]) + end + main.exit(response.status) +end \ No newline at end of file diff --git a/src/program/config/get/get.lua b/src/program/config/get/get.lua index 0e23dc7209..5c205b3e1d 100644 --- a/src/program/config/get/get.lua +++ b/src/program/config/get/get.lua @@ -9,6 +9,5 @@ function run(args) args.instance_id, 'get-config', { schema = args.schema_name, revision = args.revision_date, path = args.path }) - print(response.config) - main.exit(0) + common.print_and_exit(response, "config") end diff --git a/src/program/config/get_state/get_state.lua b/src/program/config/get_state/get_state.lua index b16eded846..ce3d29b048 100644 --- a/src/program/config/get_state/get_state.lua +++ b/src/program/config/get_state/get_state.lua @@ -4,11 +4,10 @@ module(..., package.seeall) local common = require("program.config.common") function run(args) - args = common.parse_command_line(args, {command='get-state', with_path=true}) - local response = common.call_leader( - args.instance_id, 'get-state', + args = common.parse_command_line(args, {command='get-state', with_path=true}) + local response = common.call_leader( + args.instance_id, 'get-state', { schema = args.schema_name, revision = args.revision_date, path = args.path }) - print(response.state) - main.exit(0) + common.print_and_exit(response, "state") end \ No newline at end of file diff --git a/src/program/config/load/load.lua b/src/program/config/load/load.lua index 9ef68d4c0b..ba4d49df13 100644 --- a/src/program/config/load/load.lua +++ b/src/program/config/load/load.lua @@ -11,5 +11,5 @@ function run(args) { schema = args.schema_name, revision = args.revision_date, config = common.serialize_config(args.config, args.schema_name) }) -- The reply is empty. - main.exit(0) + common.print_and_exit(response) end diff --git a/src/program/config/remove/remove.lua b/src/program/config/remove/remove.lua index 9f6ee297b3..abfba72c5f 100644 --- a/src/program/config/remove/remove.lua +++ b/src/program/config/remove/remove.lua @@ -11,5 +11,5 @@ function run(args) { schema = args.schema_name, revision = args.revision_date, path = args.path }) -- The reply is empty. - main.exit(0) + common.print_and_exit(response) end diff --git a/src/program/config/set/set.lua b/src/program/config/set/set.lua index 7e5227a827..c9077b8383 100644 --- a/src/program/config/set/set.lua +++ b/src/program/config/set/set.lua @@ -13,5 +13,5 @@ function run(args) config = common.serialize_config( args.value, args.schema_name, args.path) }) -- The reply is empty. - main.exit(0) + common.print_and_exit(response) end From 827602101956c6fc01f821e131fa3cbc1effa89e Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 20 Dec 2016 15:48:18 +0100 Subject: [PATCH 450/631] Fix small cosmetic nits brought in in review --- src/apps/config/leader.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index d2e871bb47..f317d0bfaf 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -490,7 +490,7 @@ function Leader:rpc_set_config (args) return self:handle_rpc_update_config(args, 'set', compute_set_config_fn) end local success, response = pcall(setter) - if success then return response else return {status=1, error=response} end + if success then return response else return {status=1, error=response} end end function Leader:rpc_add_config (args) @@ -542,9 +542,9 @@ function Leader:rpc_get_state (args) local printer = path_printer_for_schema_by_name(self.schema_name, args.path) local s = {} for _, follower in pairs(self.followers) do - for k,v in pairs(state.show_state(self.schema_name, follower.pid, args.path)) do + for k,v in pairs(state.show_state(self.schema_name, follower.pid, args.path)) do s[k] = v - end + end end return {state=printer(s, yang.string_output_file())} end From 71fab703dd1c45cb999c83b9cca5d6b5f9c416ee Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 20 Dec 2016 16:27:27 +0100 Subject: [PATCH 451/631] Remove .dasl extension --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index ccb295e3bc..fa192e7f8b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -40,7 +40,7 @@ EXE := bin/snabb $(patsubst %,bin/%,$(PROGRAM)) # for each module that has a top-level selftest () function. TESTMODS = $(shell find . -regex '[^\#]*\.\(lua\|dasl\)' -printf '%P ' | \ xargs grep -s -l '^function selftest *[[:punct:]]' | \ - sed -e 's_\.lua__' -e 's_/_._g') + sed -e 's_\.lua__' -e 's_\.dasl__' -e 's_/_._g') # TESTSCRIPTS expands to: # lib/watchdog/selftest.sh ... From 153386954a680cce1d25f7b043466fc970cf8476 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 21 Dec 2016 04:49:21 +0000 Subject: [PATCH 452/631] Refactor lwaftrctl - Get rid of `screen` command. - Connect to VM via telnet instead of ssh. - Change syntax to "verb command". - Remove "lwaftr" command. - Enable extra tap network interface to have external internet access from the guest. --- src/program/lwaftr/virt/lwaftrctl | 207 ++++++++---------- .../lwaftr/virt/lwaftrctl.conf.example | 37 ++-- .../lwaftr/virt/setup_networks/lwaftr1.sh | 1 + 3 files changed, 113 insertions(+), 132 deletions(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index 753d407875..86624f6d14 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -4,9 +4,8 @@ # # The scripts enables several commands: # -# * lwaftrctl snabbnfv start|stop. Starts or stops snabbnfv. -# * lwaftrctl vm start|stop. Starts or stops virtual machine -# * lwaftrctl lwaftr start|stop. Starts or stops lwaftr inside virtual machine. +# * lwaftrctl start|stop snabbnfv Starts or stops snabbnfv. +# * lwaftrctl start|stop vm Starts or stops virtual machine # # A configuration file named 'lwaftrctl.conf' must exist in the current # directory. This file contains all the variable definitions needed to run the @@ -14,89 +13,81 @@ # # The usual workflow to run the script would be the following: # -# * lwaftrctl snabbnfv start. +# * lwaftrctl start snabbnfv. # Brings up NICs in host that will be used by virtual machine. # -# * lwaftrctl vm start. +# * lwaftrctl start vm. # Brings up VM. After this step it should be possible to log into -# the VM: ssh igalia@10.21.21.2. -# -# * lwaftrctl lwaftr start. -# Starts lwaftr inside VM. This commands logs into VM and runs file -# "~/run_lwaftr.sh". See run_lwaftr.sh.example. -# -# Once the lwaftr is running within the VM, run 'lwaftr transient' from -# host to check everything is working fine. +# the VM: telnet localhost 5001. # # The command 'lwaftrctl all start', run all the steps above. -qemu_start_vm() { - echo "Starting QEMU. Please wait..." - cd $VM_DIR - sudo numactl -m ${NUMA_NODE} taskset -c ${QEMU_CORE} qemu-system-x86_64 \ - --enable-kvm -name ${NAME} -drive file=${DISK},if=virtio \ - -m ${MEM} -cpu host \ - -fsdev local,security_model=passthrough,id=fsdev0,path=${SHARED_LOCATION} \ - -device virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=share \ - -object memory-backend-file,id=mem,size=${MEM},mem-path=${HUGEPAGESTLB_FS},share=on \ - -numa node,memdev=mem \ - -chardev socket,id=char1,path=${VHU_SOCK1},server \ - -netdev type=vhost-user,id=net0,chardev=char1 \ - -device virtio-net-pci,netdev=net0,addr=0x8,mac=${SNABBNFV_MAC1} \ - -chardev socket,id=char2,path=${VHU_SOCK2},server \ - -netdev type=vhost-user,id=net1,chardev=char2 \ - -device virtio-net-pci,netdev=net1,addr=0x9,mac=${SNABBNFV_MAC2} \ - -netdev type=tap,id=net2,ifname=${IFACE},vhost=on,script=${NETSCRIPT} \ - -device virtio-net-pci,netdev=net2,addr=0xa,mac=52:54:00:00:00:03 \ - -vnc :${VNC_DISPLAY} -daemonize & -} +QEMU=qemu-system-x86_64 +QEMU_ARGS="" -vm_pid() { - echo `ps aux | grep -i "name $1" | grep -v "grep" | awk '{ print $2 }'` -} +tmux_session="" +snabb_n=0 -start_vm() { - qemu_start_vm +function run_telnet { + (echo "$2"; sleep ${3:-2}) \ + | telnet localhost $1 2>&1 } -stop_vm() { - echo "Switching off VM. Please wait..." - ssh $SSH_OPTS ${VM_USER}@${VM_IP} 'sudo halt' - sleep 10 - pid=`pidof qemu-system-x86_64` - sudo kill ${pid} 2>/dev/null - echo "VM switched off" +function tmux_launch { + local id=$1 + local command="$2 2>&1 | tee $3" + tmux_session="$id-$$" + tmux new-session -d -n "$id" -s $tmux_session "$command" } -restart_vm() { - stop_vm - start_vm +function qemu_start_vm { + echo "Starting QEMU" + cd $VM_DIR + tmux_launch \ + "qemu0" \ + "sudo numactl -m ${NUMA_NODE} taskset -c ${QEMU_CORE} \ + $QEMU $QEMU_ARGS \ + -kernel ${BZ_IMAGE} \ + -append \"earlyprintk root=/dev/vda rw console=tty0 ip=${VM_IP} hugepages=256\" \ + -M pc -smp 1 -cpu host -m ${MEM} -enable-kvm \ + -numa node,memdev=mem -object memory-backend-file,id=mem,size=${MEM},mem-path=${HUGEPAGES_FS},share=on \ + -netdev type=vhost-user,id=net0,chardev=char0 -chardev socket,id=char0,path=${SNABBNFV_SOCK1},server \ + -device virtio-net-pci,netdev=net0,addr=0x8,mac=${SNABBNFV_MAC1} \ + -netdev type=vhost-user,id=net1,chardev=char1 -chardev socket,id=char1,path=${SNABBNFV_SOCK2},server \ + -device virtio-net-pci,netdev=net1,addr=0x9,mac=${SNABBNFV_MAC2} \ + -netdev type=tap,id=net2,ifname=${IFACE},vhost=on,script=${NETSCRIPT} \ + -device virtio-net-pci,netdev=net2,addr=0xa,mac=52:54:00:00:00:03 \ + -serial telnet:localhost:${VM_PORT},server,nowait \ + -display none \ + -drive if=virtio,format=${VM_FORMAT},file=${VM_IMAGE} " \ + "qemu0.log" + echo "Connect via 'telnet localhost ${VM_PORT}'" } -start_lwaftr() { - echo "Start lwAFTR" - - ssh $SSH_OPTS ${VM_USER}@${VM_IP} "~/run_lwaftr.sh" +function start_vm { + qemu_start_vm } -stop_lwaftr() { - echo "Stop lwAFTR" - - ssh $SSH_OPTS ${VM_USER}@${VM_IP} 'sudo killall snabb 2>/dev/null' +function stop_vm { + echo "Stopping VM" + run_telnet ${VM_PORT} "sudo poweroff" >/dev/null + pid=$(pidof $QEMU) + sudo kill ${pid} 2>/dev/null + echo "Done" } -restart_lwaftr() { - stop_lwaftr - start_lwaftr +function restart_vm { + stop_vm + start_vm } -start_snabbnfv() { - start_snabbnfv_process "snabbnfv-1" $SNABBNFV_CORE1 $SNABBNFV_PCI1 $SNABBNFV_CONF1 $VHU_SOCK1 - start_snabbnfv_process "snabbnfv-2" $SNABBNFV_CORE2 $SNABBNFV_PCI2 $SNABBNFV_CONF2 $VHU_SOCK2 +function start_snabbnfv { + start_snabbnfv_process "snabbnfv-1" $SNABBNFV_CORE1 $SNABBNFV_PCI1 $SNABBNFV_CONF1 $SNABBNFV_SOCK1 + start_snabbnfv_process "snabbnfv-2" $SNABBNFV_CORE2 $SNABBNFV_PCI2 $SNABBNFV_CONF2 $SNABBNFV_SOCK2 } -start_snabbnfv_process() { - local screen=$1 +function start_snabbnfv_process { + local id=$1 local core=$2 local pci=$3 local conf=$4 @@ -107,65 +98,66 @@ start_snabbnfv_process() { echo "WARNING: File '${conf}' does not exist." fi - # Run snabbnfv inside screen. - screen -L -dmS $screen bash -c \ - "cd ${SNABB_HOST_PATH}/src; \ - sudo numactl -m ${NUMA_NODE} taskset -c ${core} ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock};" + # Run snabbnfv inside tmux session. + tmux_launch "$id" \ + "cd ${SNABB_PATH}/src; sudo numactl -m ${NUMA_NODE} taskset -c ${core} ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock}" \ + $(snabb_log) + status=$(tmux ls | grep -c "$id") sleep 0.5 - status=$(screen -ls | grep ${screen}) # Check exit status. - if [[ -z ${status} ]]; then + if [[ ${status} -eq 0 ]]; then echo "Start of snabbnfv failed: " echo -e "\tsudo numactl -m ${NUMA_NODE} taskset -c ${core} ./snabb snabbnfv traffic -b ${pci} ${conf} ${sock}" exit 1 fi - echo "Started snabbnfv on core ${core} (screen: '$screen')" + echo "Started snabbnfv on core ${core} (tmux: '$id')" echo -e "\t{PCI: ${pci}, conf: ${conf}, sock: ${sock}}" } -restart_snabbnfv() { - stop_snabbnfv - start_snabbnfv +function snabb_log { + echo "snabb${snabb_n}.log" } -kill_all() { +function stop_snabbnfv { + echo "Stopping snabbnfv" + kill_all "snabbnfv traffic" + remove_file $SNABBNFV_SOCK1 + remove_file $SNABBNFV_SOCK2 + echo "Done" +} + +function kill_all { local name=$1 pids=`ps aux | grep "$name" | awk '{ print $2 }'` for pid in ${pids[@]}; do sudo kill $pid 2>/dev/null done sleep 1 - screen -wipe } -remove_file() { +function remove_file { sudo rm -f $1 } -stop_snabbnfv() { - echo "Stop snabbnfv" - - kill_all "snabbnfv traffic" - remove_file $VHU_SOCK1 - remove_file $VHU_SOCK2 +function restart_snabbnfv { + stop_snabbnfv + start_snabbnfv } -start_command() { +# Actions. + +function start_command { COMMAND=$1 case $COMMAND in "all") start_snabbnfv start_vm - start_lwaftr ;; "snabbnfv") start_snabbnfv ;; - "lwaftr") - start_lwaftr - ;; "vm") start_vm ;; @@ -175,21 +167,18 @@ start_command() { esac } -restart_command() { +function stop_command { COMMAND=$1 case $COMMAND in "all") - restart_vm - restart_lwaftr + stop_vm + stop_snabbnfv ;; "snabbnfv") - restart_snabbnfv - ;; - "lwaftr") - restart_lwaftr + stop_snabbnfv ;; "vm") - restart_vm + stop_vm ;; *) bad_usage @@ -197,22 +186,18 @@ restart_command() { esac } -stop_command() { +function restart_command { COMMAND=$1 case $COMMAND in "all") - stop_lwaftr - stop_vm - stop_snabbnfv + restart_snabbnfv + restart_vm ;; "snabbnfv") - stop_snabbnfv - ;; - "lwaftr") - stop_lwaftr + restart_snabbnfv ;; "vm") - stop_vm + restart_vm ;; *) bad_usage @@ -222,17 +207,17 @@ stop_command() { # Main -usage() { +function usage { local exit_code=$1 - echo "Usage: lwaftrctl -f lwaftrctl.conf [all|snabbnfv|vm|lwaftr] [start|stop|restart]" + echo "Usage: lwaftrctl -f lwaftrctl.conf [start|stop|restart] [all|snabbnfv|vm]" exit $exit_code } -bad_usage() { +function bad_usage { usage -1 } -help() { +function help { usage 0 } @@ -265,8 +250,8 @@ fi source $AFTRCTL_CONF PROGRAM_NAME=$0 -COMMAND=${ARGS[0]} -ACTION=${ARGS[1]} +ACTION=${ARGS[0]} +COMMAND=${ARGS[1]} case $ACTION in "start") diff --git a/src/program/lwaftr/virt/lwaftrctl.conf.example b/src/program/lwaftr/virt/lwaftrctl.conf.example index edf1e3c364..a5bfec8052 100644 --- a/src/program/lwaftr/virt/lwaftrctl.conf.example +++ b/src/program/lwaftr/virt/lwaftrctl.conf.example @@ -2,44 +2,39 @@ # General -SNABB_HOST_PATH=~/workspace/snabb_host -SNABB_HOST_LWAFTR=$SNABB_HOST_PATH/src/program/lwaftr +SNABB_PATH=~/workspace/snabb +SNABB_LWAFTR=$SNABB_PATH/src/program/lwaftr # QEMU NAME=lwaftr1 -DISK=~/vm/vm_lwaftr1.qcow2 -MEM=1024M -IFACE=mgmt0 -NETSCRIPT=$SNABB_HOST_LWAFTR/virt/setup_networks/lwaftr1.sh +NUMA_NODE=0 +QEMU_CORE=0 -SHARED_LOCATION=~/workspace -VHU_SOCK1=/tmp/vh1a.sock -VHU_SOCK2=/tmp/vh1b.sock -VNC_DISPLAY=22 +MEM=2G +HUGEPAGES_FS=/dev/hugepages -# VM +BZ_IMAGE=~/.test_env/bzImage +VM_IMAGE=~/.test_env/lwaftr-vm.img -HUGEPAGESTLB_FS=/dev/hugepages -NUMA_NODE=0 -QEMU_CORE=0 -VM_DIR=~/vm +VM_PORT=5001 +VM_FORMAT=raw VM_IP=10.21.21.2 -VM_USER=igalia # Must be in the list of sudoers - -# Guest -SNABB_GUEST_PATH=/mnt/host/snabb_guest/src +IFACE=mgmt0 +NETSCRIPT=$SNABB_LWAFTR/virt/setup_networks/lwaftr1.sh # SnabbNFV SNABBNFV_CORE1=1 -SNABBNFV_CONF1=$SNABB_HOST_LWAFTR/virt/ports/lwaftr1/a.cfg +SNABBNFV_CONF1=$SNABB_LWAFTR/virt/ports/lwaftr1/a.cfg SNABBNFV_PCI1=0000:02:00.0 SNABBNFV_MAC1=52:54:00:00:00:01 +SNABBNFV_SOCK1=/tmp/vh1a.sock SNABBNFV_CORE2=2 -SNABBNFV_CONF2=$SNABB_HOST_LWAFTR/virt/ports/lwaftr1/b.cfg +SNABBNFV_CONF2=$SNABB_LWAFTR/virt/ports/lwaftr1/b.cfg SNABBNFV_PCI2=0000:02:00.1 SNABBNFV_MAC2=52:54:00:00:00:02 +SNABBNFV_SOCK2=/tmp/vh1b.sock diff --git a/src/program/lwaftr/virt/setup_networks/lwaftr1.sh b/src/program/lwaftr/virt/setup_networks/lwaftr1.sh index 5437f7e4af..f502fc72d8 100755 --- a/src/program/lwaftr/virt/setup_networks/lwaftr1.sh +++ b/src/program/lwaftr/virt/setup_networks/lwaftr1.sh @@ -7,6 +7,7 @@ if [ -n "$1" ] then sleep 0.5s if [ "$1" = ${IFACE} ]; then + ip li set up dev ${IFACE} ip addr add ${IP} dev ${IFACE} sleep 0.5s fi; From c97fcae2b5f9aeeb798ebaddc14386a67791f462 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 29 Dec 2016 18:22:27 +0100 Subject: [PATCH 453/631] Execute make clean in programs containing a Makefile - program/lwaftr/Makefile: Clean up .o files. - program/snabbvmx/Makefile: Clean up .o files. --- src/Makefile | 13 ++++++++++--- src/program/lwaftr/Makefile | 8 +++++--- src/program/snabbvmx/Makefile | 8 +++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/Makefile b/src/Makefile index 9e956d696b..e5542497cc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -209,7 +209,15 @@ obj/doc/snabb.epub: obj/doc/snabb.markdown CLEAN = snabb obj bin testlog programs.inc -clean: +clean_programs: + @(for d in program/*/; do \ + if [ -f "$$d/Makefile" ]; then \ + echo "CLEAN $$d"; \ + make -s -C $$d clean; \ + fi \ + done) + +clean: clean_programs $(E) "RM $(CLEAN)" $(Q)-rm -rf $(CLEAN) @@ -220,5 +228,4 @@ mrproper: clean benchmarks: $(Q) (scripts/bench.sh) -.PHONY: clean $(TESTMODS) $(TESTSCRIPTS) benchmarks - +.PHONY: clean_programs clean $(TESTMODS) $(TESTSCRIPTS) benchmarks diff --git a/src/program/lwaftr/Makefile b/src/program/lwaftr/Makefile index fa1f12b3c4..73d7c4527f 100644 --- a/src/program/lwaftr/Makefile +++ b/src/program/lwaftr/Makefile @@ -24,9 +24,11 @@ doc/snabb-lwaftr.epub: doc/snabb-lwaftr.md CLEAN = doc/snabb-lwaftr.* -clean: +clean_object_files: + @(find . -name "*.o" -exec echo "RM {}" \; -exec rm -f {} \;) + +clean: clean_object_files $(E) "RM $(CLEAN)" $(Q)-rm -rf $(CLEAN) - $(Q) (cd ../../; make clean) -.PHONY: clean +.PHONY: clean_object_files clean diff --git a/src/program/snabbvmx/Makefile b/src/program/snabbvmx/Makefile index 7b4a285349..548db6e5c4 100644 --- a/src/program/snabbvmx/Makefile +++ b/src/program/snabbvmx/Makefile @@ -20,9 +20,11 @@ doc/snabbvmx.epub: doc/snabbvmx.md CLEAN = doc/snabbvmx.* -clean: +clean_object_files: + @(find . -name "*.o" -exec echo "RM {}" \; -exec rm -f {} \;) + +clean: clean_object_files $(E) "RM $(CLEAN)" $(Q)-rm -rf $(CLEAN) - $(Q) (cd ../../; make clean) -.PHONY: clean +.PHONY: clean_object_files clean From e96d016ff92831b274e574ae4418e7f1351b4feb Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 2 Jan 2017 12:15:23 +0000 Subject: [PATCH 454/631] Reintroduce lwaftr command --- src/program/lwaftr/virt/lwaftrctl | 34 +++++++++++++++++-- src/program/lwaftr/virt/run_lwaftr.sh.example | 9 ----- .../lwaftr/virt/start-lwaftr.sh.example | 10 ++++++ .../lwaftr/virt/stop-lwaftr.sh.example | 3 ++ 4 files changed, 45 insertions(+), 11 deletions(-) delete mode 100644 src/program/lwaftr/virt/run_lwaftr.sh.example create mode 100644 src/program/lwaftr/virt/start-lwaftr.sh.example create mode 100644 src/program/lwaftr/virt/stop-lwaftr.sh.example diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index 86624f6d14..9ef11704c3 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -24,6 +24,7 @@ QEMU=qemu-system-x86_64 QEMU_ARGS="" +LWAFTR_DELAY=${LWAFTR_DELAY:-"20"} tmux_session="" snabb_n=0 @@ -41,7 +42,7 @@ function tmux_launch { } function qemu_start_vm { - echo "Starting QEMU" + echo "Started QEMU" cd $VM_DIR tmux_launch \ "qemu0" \ @@ -61,7 +62,6 @@ function qemu_start_vm { -display none \ -drive if=virtio,format=${VM_FORMAT},file=${VM_IMAGE} " \ "qemu0.log" - echo "Connect via 'telnet localhost ${VM_PORT}'" } function start_vm { @@ -146,6 +146,22 @@ function restart_snabbnfv { start_snabbnfv } +function start_lwaftr { + echo "Started lwaftr" + run_telnet ${VM_PORT} "sudo ~/start-lwaftr.sh" >/dev/null +} + +function stop_lwaftr { + echo "Stopping lwaftr" + run_telnet ${VM_PORT} "sudo ~/stop-lwaftr.sh" >/dev/null + echo "Done" +} + +function restart_lwaftr { + stop_lwaftr + start_lwaftr +} + # Actions. function start_command { @@ -154,12 +170,19 @@ function start_command { "all") start_snabbnfv start_vm + echo "Waiting ${LWAFTR_DELAY} seconds" + sleep $LWAFTR_DELAY + start_lwaftr ;; "snabbnfv") start_snabbnfv ;; "vm") start_vm + echo "Connect via 'telnet localhost ${VM_PORT}'" + ;; + "lwaftr") + start_lwaftr ;; *) bad_usage @@ -171,6 +194,7 @@ function stop_command { COMMAND=$1 case $COMMAND in "all") + stop_lwaftr stop_vm stop_snabbnfv ;; @@ -180,6 +204,9 @@ function stop_command { "vm") stop_vm ;; + "lwaftr") + stop_lwaftr + ;; *) bad_usage ;; @@ -199,6 +226,9 @@ function restart_command { "vm") restart_vm ;; + "lwaftr") + restart_lwaftr + ;; *) bad_usage ;; diff --git a/src/program/lwaftr/virt/run_lwaftr.sh.example b/src/program/lwaftr/virt/run_lwaftr.sh.example deleted file mode 100644 index 02eea9b699..0000000000 --- a/src/program/lwaftr/virt/run_lwaftr.sh.example +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/evn bash - -SNABB_GUEST=/mnt/host/snabb_guest/src -V4_PCI=0000:00:08.0 -V6_PCI=0000:00:09.0 -LWAFTR_CONF=program/lwaftr/tests/data/icmp_on_fail.conf -OUTPUT=/tmp/lwaftr.csv - -screen -dmS lwaftr bash -c "cd ${SNABB_GUEST};sudo taskset -c 1 ./snabb snsh -p lwaftr run -v -i --conf $LWAFTR_CONF --v4 ${V4_PCI} --v6 ${V6_PCI} | tee $OUTPUT" diff --git a/src/program/lwaftr/virt/start-lwaftr.sh.example b/src/program/lwaftr/virt/start-lwaftr.sh.example new file mode 100644 index 0000000000..ec2199bf23 --- /dev/null +++ b/src/program/lwaftr/virt/start-lwaftr.sh.example @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +SNABB_PCI0=0000:00:08.0 +SNABB_PCI1=0000:00:09.0 +SNABB_EXE=/root/snabb/src/snabb +LWAFTR_CONF=/root/lwaftr-migrated.conf + +sudo ${SNABB_EXE} lwaftr run --virtio \ + --conf ${LWAFTR_CONF} \ + --v4 ${SNABB_PCI0} --v6 ${SNABB_PCI1} diff --git a/src/program/lwaftr/virt/stop-lwaftr.sh.example b/src/program/lwaftr/virt/stop-lwaftr.sh.example new file mode 100644 index 0000000000..ba258b17f6 --- /dev/null +++ b/src/program/lwaftr/virt/stop-lwaftr.sh.example @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +ps aux | grep "snabb lwaftr run" | awk '{ print $2; }' | xargs kill From 17733104069d2bf304e58201a4a3353d8bcbad68 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 3 Jan 2017 01:33:54 +0100 Subject: [PATCH 455/631] Replace tabs for spaces --- src/lib/yang/parser.lua | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 023e627af6..af26e32cf0 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -122,8 +122,8 @@ function Parser:skip_whitespace() if self:check("*") then self:skip_c_comment() else - self:consume("/") - self:take_while('[^\n]') + self:consume("/") + self:take_while('[^\n]') end self:take_while('%s') end @@ -154,21 +154,21 @@ function Parser:parse_qstring(quote) result = result..self:take_while("[^"..terminators.."]") if self:check(quote) then break end if self:check("\n") then - while self.column < start_column do - if not self:check(" ") and not self:check("\t") then break end - end - result = result.."\n" - if self.column > start_column then - result = result..string.rep(" ", self.column-start_column) - end + while self.column < start_column do + if not self:check(" ") and not self:check("\t") then break end + end + result = result.."\n" + if self.column > start_column then + result = result..string.rep(" ", self.column-start_column) + end elseif self:check("\\") then - if self:check("n") then result = result.."\n" - elseif self:check("t") then result = result.."\t" - elseif self:check('"') then result = result..'"' - elseif self:check("\\") then result = result.."\\" - else - result = result.."\\" - end + if self:check("n") then result = result.."\n" + elseif self:check("t") then result = result.."\t" + elseif self:check('"') then result = result..'"' + elseif self:check("\\") then result = result.."\\" + else + result = result.."\\" + end end end @@ -232,7 +232,7 @@ function Parser:parse_statement_list() while true do self:skip_whitespace() if self:is_eof() or self:peek() == "}" then - break + break end table.insert(statements, self:parse_statement()) end @@ -399,9 +399,9 @@ function selftest() test_module("type/** hellooo */string;", {{keyword="type", argument="string"}}) test_module('type "hello\\pq";', {{keyword="type", argument="hello\\pq"}}) test_module(lines("leaf port {", "type number;", "}"), {{keyword="leaf", - argument="port", statements={{keyword="type", argument="number"}}}}) + argument="port", statements={{keyword="type", argument="number"}}}}) test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", - argument="port", statements={{keyword="type"}}}}) + argument="port", statements={{keyword="type"}}}}) parse(require('lib.yang.ietf_inet_types_yang')) parse(require('lib.yang.ietf_yang_types_yang')) From c00f2c3911bad7643d9295ca3fbed5e7f605117f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 3 Jan 2017 01:35:03 +0100 Subject: [PATCH 456/631] Remove unneeded check --- src/lib/yang/parser.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index af26e32cf0..c22b4844cc 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -145,7 +145,6 @@ end function Parser:parse_qstring(quote) local start_column = self.column - self:check(quote) local terminators = "\n"..quote if quote == '"' then terminators = terminators.."\\" end @@ -398,6 +397,7 @@ function selftest() test_module("// foo bar;\nleaf port;", {{keyword="leaf", argument="port"}}) test_module("type/** hellooo */string;", {{keyword="type", argument="string"}}) test_module('type "hello\\pq";', {{keyword="type", argument="hello\\pq"}}) + test_module('description "";', {{keyword="description"}}) test_module(lines("leaf port {", "type number;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type", argument="number"}}}}) test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", From b779a2b11a63e6a800f9b7f651e3137ba62ea188 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 3 Jan 2017 01:41:57 +0100 Subject: [PATCH 457/631] Clean up --- src/lib/yang/parser.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index c22b4844cc..5874037e99 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -168,7 +168,6 @@ function Parser:parse_qstring(quote) else result = result.."\\" end - end end self:check(quote) @@ -177,7 +176,7 @@ function Parser:parse_qstring(quote) if not self:check("+") then return result end self:skip_whitespace() - -- Strings can be concaternated together with a + + -- Strings can be concatenated together with a + if self:check("'") then return result..self:parse_qstring("'") elseif self:check('"') then From fb522036ee4686dce413cc9e67d49422f5a1bdd6 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 30 Dec 2016 10:18:39 +0000 Subject: [PATCH 458/631] Fix tunnel-path-mtu and tunnel-payload-mtu in ietf-softwire --- src/apps/config/support/snabb-softwire-v1.lua | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 656f18bb4b..d4a0fc5d46 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -219,9 +219,9 @@ local function ietf_softwire_translator () local br_instance, br_instance_key_t = cltable_for_grammar(get_ietf_br_instance_grammar()) br_instance[br_instance_key_t({id=1})] = { + tunnel_payload_mtu = native_config.softwire_config.internal_interface.mtu, + tunnel_path_mru = native_config.softwire_config.external_interface.mtu, -- FIXME - tunnel_payload_mtu = 0, - tunnel_path_mru = 0, softwire_num_threshold = 0xffffffff, binding_table = { binding_entry = ietf_binding_table_from_native( @@ -275,6 +275,21 @@ local function ietf_softwire_translator () local br_instance_paths = {'softwire-config', 'binding', 'br', 'br-instances', 'br-instance'} local bt_paths = {'binding-table', 'binding-entry'} + + -- Handle special br attributes (tunnel-payload-mtu, tunnel-path-mru) + if #path > #br_instance_paths then + if path[#path].name == 'tunnel-payload-mtu' then + return {{'set', {schema='snabb-softwire-v1', + path="/softwire-config/internal-interface/mtu", + config=tostring(arg)}}} + end + if path[#path].name == 'tunnel-path-mru' then + return {{'set', {schema='snabb-softwire-v1', + path="/softwire-config/external-interface/mtu", + config=tostring(arg)}}} + end + end + -- Two kinds of updates: setting the whole binding table, or -- updating one entry. if sets_whole_table(path, #br_instance_paths + #bt_paths) then From 86b9835eed86b0068d9e93637cc175167f1feba1 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 3 Jan 2017 23:08:37 +0000 Subject: [PATCH 459/631] Remove end-addr in psid-map --- src/apps/lwaftr/binding_table.lua | 2 +- .../lwaftr/migrate_configuration/migrate_configuration.lua | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index 99a252b3cc..d8260454ce 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -260,7 +260,7 @@ function load(conf) 'psid_length '..psid_length..' + shift '..shift.. ' should not exceed 16') psid_value.psid_length, psid_value.shift = psid_length, shift - psid_builder:add_range(k.addr, v.end_addr or k.addr, psid_value) + psid_builder:add_range(k.addr, k.addr, psid_value) end local psid_map = psid_builder:build(psid_map_value_t()) return BindingTable.new(psid_map, conf.br_address, conf.softwire) diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index d2062ecfaf..7316841fab 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -320,10 +320,8 @@ local function migrate_conf(old) local psid_map = cltable.new({ key_type = psid_key_t }) for addr, end_addr, params in old_bt.psid_map:iterate() do local reserved_ports_bit_count = 16 - params.psid_length - params.shift - if end_addr == addr then end_addr = nil end if reserved_ports_bit_count ~= 16 then psid_map[psid_key_t(addr)] = { - end_addr = end_addr, psid_length = params.psid_length, shift = params.shift, reserved_ports_bit_count = reserved_ports_bit_count From b3fdcaf1c6eed7c379fb1d1ec9177d69f7fec5ef Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 3 Jan 2017 23:21:57 +0000 Subject: [PATCH 460/631] Remove end-addr from configuration files --- src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf | 1 - src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf | 1 - .../snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf | 1 - .../snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf | 1 - 4 files changed, 4 deletions(-) diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf index e60687dfd3..84674e15b3 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr-xe0.conf @@ -3,7 +3,6 @@ softwire-config { br-address fc00::100; psid-map { addr 193.5.1.100; - end-addr 193.5.1.102; psid-length 6; shift 10; } diff --git a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf index e60687dfd3..84674e15b3 100644 --- a/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf +++ b/src/program/snabbvmx/tests/conf/snabbvmx-lwaftr.conf @@ -3,7 +3,6 @@ softwire-config { br-address fc00::100; psid-map { addr 193.5.1.100; - end-addr 193.5.1.102; psid-length 6; shift 10; } diff --git a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf index 2b9602cd94..c928168ce2 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/snabbvmx-lwaftr-xe1.conf @@ -8,7 +8,6 @@ softwire-config { } psid-map { addr 10.10.0.0; - end-addr 10.10.0.1; psid-length 6; shift 10; } diff --git a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf index 2e083887ca..cd73839d46 100644 --- a/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf +++ b/src/program/snabbvmx/tests/end-to-end/data/vlan/snabbvmx-lwaftr-xe1.conf @@ -8,7 +8,6 @@ softwire-config { } psid-map { addr 10.10.0.0; - end-addr 10.10.0.1; psid-length 6; shift 10; } From 73c6590d4869faefc42a81c5ac3617734429e14a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 6 Jan 2017 17:37:23 +0100 Subject: [PATCH 461/631] Add socket support for snabb config listen --- src/program/config/common.lua | 7 +++--- src/program/config/listen/README | 5 ++++ src/program/config/listen/listen.lua | 37 +++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 4250053a9f..834e7755ab 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -43,9 +43,10 @@ function parse_command_line(args, opts) function handlers.h() show_usage(opts.command, 0) end function handlers.s(arg) ret.schema_name = arg end function handlers.r(arg) ret.revision_date = arg end - args = lib.dogetopt(args, handlers, "hs:r:", + function handlers.c(arg) ret.socket = arg end + args = lib.dogetopt(args, handlers, "hs:r:c:", {help="h", ['schema-name']="s", schema="s", - ['revision-date']="r", revision="r"}) + ['revision-date']="r", revision="r", socket="c"}) if #args == 0 then err() end ret.instance_id = table.remove(args, 1) local descr = call_leader(ret.instance_id, 'describe', {}) @@ -149,4 +150,4 @@ function print_and_exit(response, response_prop) print(response[response_prop]) end main.exit(response.status) -end \ No newline at end of file +end diff --git a/src/program/config/listen/README b/src/program/config/listen/README index f2d78464ff..585d7e2f94 100644 --- a/src/program/config/listen/README +++ b/src/program/config/listen/README @@ -5,9 +5,14 @@ Available options: -s, --schema SCHEMA YANG data interface to request. -r, --revision REVISION Require a specific revision of the YANG module. -h, --help Display this message. + -c, --socket Path to unix socket if using. If the --schema argument is not provided, "snabb config" will ask the data plane for its native schema. +If the --socket argument is provided, it will accept a single client over the +unix socket. If it has not been provided, it will accept commands over stdin +and display them on stdout. + See https://github.com/Igalia/snabb/blob/lwaftr/src/program/config/README.md for full documentation. diff --git a/src/program/config/listen/listen.lua b/src/program/config/listen/listen.lua index 47d29b007c..413a04072b 100644 --- a/src/program/config/listen/listen.lua +++ b/src/program/config/listen/listen.lua @@ -9,6 +9,16 @@ local path_lib = require("lib.yang.path") local common = require("program.config.common") local json_lib = require("program.config.json") +local function open_socket(file) + S.signal('pipe', 'ign') + local socket = assert(S.socket("unix", "stream")) + S.unlink(file) + local sa = S.t.sockaddr_un(file) + assert(socket:bind(sa)) + assert(socket:listen()) + return socket +end + local function validate_value(schema_name, revision_date, path, value_str) local parser = common.data_parser(schema_name, path) local value = parser(value_str) @@ -49,14 +59,14 @@ local function read_request(client, schema_name, revision_date) path = path_lib.normalize_path(path) local handler = assert(request_handlers[data.normalize_id(verb)]) local req = handler(schema_name, revision_date, path, json.value) - local function print_reply(reply) + local function print_reply(reply, fd) local output = json_lib.buffered_output() local value if verb == 'get' then value = reply.config elseif verb == 'get-state' then value = reply.state end json_lib.write_json_object(output, {id=id, status='ok', value=value}) - output:flush(S.stdout) + output:flush(fd or S.stdout) end return req, print_reply end @@ -73,7 +83,26 @@ function run(args) local caller = rpc.prepare_caller('snabb-config-leader-v1') local leader = common.open_socket_or_die(args.instance_id) attach_listener(leader, caller, args.schema_name, args.revision_date) - local client = json_lib.buffered_input(S.stdin) + + -- Check if there is a socket path specified, if so use that as method + -- to communicate, otherwise use stdin and stdout. + local fd = nil + if args.socket then + local sockfd = open_socket(args.socket) + local addr = S.t.sockaddr_un() + -- Wait for a connection + local err + print("Listening for clients on socket: "..args.socket) + fd, err = sockfd:accept(addr) + if fd == nil then + sockfd:close() + error(err) + end + else + fd = S.stdin + end + + local client = json_lib.buffered_input(fd) local pollfds = S.types.t.pollfds({ {fd=leader, events="in"}, {fd=client, events="in"}}) @@ -119,7 +148,7 @@ function run(args) local msg, parse_reply = rpc.prepare_call( caller, request.method, request.args) local function have_reply(msg) - return print_reply(parse_reply(msg)) + return print_reply(parse_reply(msg), fd) end common.send_message(leader, msg) table.insert(pending_replies, 1, have_reply) From 5b1986d4030810524f8863aabd3373019580098e Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Wed, 11 Jan 2017 10:16:42 +0100 Subject: [PATCH 462/631] Fix lwaftr query - Make specifying pids work again - Make specifying strings containing '-' work --- src/program/lwaftr/query/query.lua | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 929cf06e6e..f718b79796 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -66,8 +66,12 @@ local function read_counters (tree) return ret, max_width end +-- Filters often contain '-', which is a special character for match. +-- Escape it. local function skip_counter (name, filter) - return filter and not name:match(filter) + local escaped_filter = filter + if escaped_filter then escaped_filter = filter:gsub("-", "%%-") end + return filter and not name:match(escaped_filter) end local function print_counter (name, value, max_width) @@ -90,7 +94,7 @@ end function run (raw_args) local opts, pid, counter_name = parse_args(raw_args) if tostring(pid) and not counter_name then - counter_name, pid = pid, nil + counter_name, pid = nil, pid end if opts.name then local programs = engine.enumerate_named_programs(opts.name) From dd8ef6cf2055dfacf0210a6f4462fff7ce734afb Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 11 Jan 2017 10:53:16 +0000 Subject: [PATCH 463/631] Add selftest --- src/program/lwaftr/query/selftest.sh | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 src/program/lwaftr/query/selftest.sh diff --git a/src/program/lwaftr/query/selftest.sh b/src/program/lwaftr/query/selftest.sh new file mode 100755 index 0000000000..71beb174d6 --- /dev/null +++ b/src/program/lwaftr/query/selftest.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +SKIPPED_CODE=43 + +if [[ -z "$SNABB_PCI0" ]]; then + echo "SNABB_PCI0 not set" + exit $SKIPPED_CODE +fi + +if [[ -z "$SNABB_PCI1" ]]; then + echo "SNABB_PCI1 not set" + exit $SKIPPED_CODE +fi + +LWAFTR_CONF=./program/lwaftr/tests/data/no_icmp.conf + +function tmux_launch { + command="$2 2>&1 | tee $3" + if [ -z "$tmux_session" ]; then + tmux_session=test_env-$$ + tmux new-session -d -n "$1" -s $tmux_session "$command" + else + tmux new-window -a -d -n "$1" -t $tmux_session "$command" + fi +} + +function kill_lwaftr { + ps aux | grep $SNABB_PCI0 | awk '{print $2}' | xargs kill 2>/dev/null +} + +function cleanup { + kill_lwaftr + exit +} + +trap cleanup EXIT HUP INT QUIT TERM + +function get_lwaftr_instance { + pids=$(ps aux | grep $SNABB_PCI0 | awk '{print $2}') + for pid in ${pids[@]}; do + if [[ -d "/var/run/snabb/$pid/apps/lwaftr" ]]; then + echo $pid + fi + done +} + +function fatal { + local msg=$1 + echo "Error: $msg" + exit 1 +} + +function test_lwaftr_query { + local pid=$1 + # FIXME: Sometimes lwaftr query gets stalled. Add timeout. + local lineno=`timeout 1 ./snabb lwaftr query $pid | wc -l` + if [[ $lineno -gt 1 ]]; then + echo "Success: lwaftr query $pid" + else + fatal "lwaftr query $pid" + fi +} + +function test_lwaftr_query_filter { + local pid=$1 + local filter=$2 + local lineno=`timeout 1 ./snabb lwaftr query $pid $filter | wc -l` + if [[ $lineno -gt 1 ]]; then + echo "Success: lwaftr query $pid $filter" + else + fatal "lwaftr query $pid" + fi +} + +# Run lwAFTR. +tmux_launch "lwaftr" "./snabb lwaftr run --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" +sleep 2 + +# Run tests. +pid=$(get_lwaftr_instance) +if [[ -n "$pid" ]]; then + test_lwaftr_query $pid + test_lwaftr_query_filter $pid "memuse" +fi From 6845bcab9fb76b94671c3537264460b683086d96 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 3 Jan 2017 15:47:05 +0000 Subject: [PATCH 464/631] Fix error on valid 'snabb config set' invocation --- src/apps/config/support/snabb-softwire-v1.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index d4a0fc5d46..f1e5a52254 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -221,7 +221,7 @@ local function ietf_softwire_translator () br_instance[br_instance_key_t({id=1})] = { tunnel_payload_mtu = native_config.softwire_config.internal_interface.mtu, tunnel_path_mru = native_config.softwire_config.external_interface.mtu, - -- FIXME + -- FIXME: What it should map to? softwire_num_threshold = 0xffffffff, binding_table = { binding_entry = ietf_binding_table_from_native( @@ -276,7 +276,7 @@ local function ietf_softwire_translator () 'br-instances', 'br-instance'} local bt_paths = {'binding-table', 'binding-entry'} - -- Handle special br attributes (tunnel-payload-mtu, tunnel-path-mru) + -- Handle special br attributes (tunnel-payload-mtu, tunnel-path-mru, softwire-num-threshold) if #path > #br_instance_paths then if path[#path].name == 'tunnel-payload-mtu' then return {{'set', {schema='snabb-softwire-v1', @@ -288,6 +288,11 @@ local function ietf_softwire_translator () path="/softwire-config/external-interface/mtu", config=tostring(arg)}}} end + if path[#path].name == 'softwire-num-threshold' then + -- FIXME: Do nothing. + return {} + end + error('unrecognized leaf: '..path[#path].name) end -- Two kinds of updates: setting the whole binding table, or From 6f806a9ee33b3d835e7d5c0aa7e2104d42a42ef7 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Thu, 12 Jan 2017 14:50:54 +0100 Subject: [PATCH 465/631] Disabled finding default Snabb instance Also added tests and updated the README/help. --- src/program/lwaftr/query/README | 7 ++++++- src/program/lwaftr/query/query.lua | 2 ++ src/program/lwaftr/query/selftest.sh | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/query/README b/src/program/lwaftr/query/README index e186e82eb6..1485e1ee14 100644 --- a/src/program/lwaftr/query/README +++ b/src/program/lwaftr/query/README @@ -1,5 +1,5 @@ Usage: - query [OPTIONS] [] [] + query [OPTIONS] [] Options: @@ -21,3 +21,8 @@ The values for the counters defined in will be displayed, but only the ones that are not zero. It needs root privileges. + +Examples: +snabb lwaftr query -l +snabb lwaftr query 1111 in-ipv4 +snabb lwaftr query -n mysnabbprocess diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index f718b79796..15d3aa803c 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -97,12 +97,14 @@ function run (raw_args) counter_name, pid = nil, pid end if opts.name then + -- without --reconfigurable local programs = engine.enumerate_named_programs(opts.name) pid = programs[opts.name] if not pid then fatal(("Couldn't find process with name '%s'"):format(opts.name)) end end + if not pid then fatal("No pid or name specified") end local instance_tree = top.select_snabb_instance(pid) print_counters(instance_tree, counter_name) end diff --git a/src/program/lwaftr/query/selftest.sh b/src/program/lwaftr/query/selftest.sh index 71beb174d6..c550f9f9ed 100755 --- a/src/program/lwaftr/query/selftest.sh +++ b/src/program/lwaftr/query/selftest.sh @@ -73,12 +73,14 @@ function test_lwaftr_query_filter { } # Run lwAFTR. -tmux_launch "lwaftr" "./snabb lwaftr run --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" +tmux_launch "lwaftr" "./snabb lwaftr run --reconfigurable --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" sleep 2 # Run tests. pid=$(get_lwaftr_instance) if [[ -n "$pid" ]]; then test_lwaftr_query $pid + test_lwaftr_query $pid -l test_lwaftr_query_filter $pid "memuse" + test_lwaftr_query_filter $pid "in-ipv4" fi From a9d3130d0a955f5a8424e3cf7378c0e7d6434ae2 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Thu, 12 Jan 2017 15:21:46 +0100 Subject: [PATCH 466/631] Fix byname printing of stats with --reconfigurable --- src/program/lwaftr/query/query.lua | 22 +++++++++++++++++++++- src/program/ps/ps.lua | 6 +++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 15d3aa803c..43f5c75d5f 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -7,6 +7,8 @@ local lwcounter = require("apps.lwaftr.lwcounter") local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") local top = require("program.top.top") +local app = require("core.app") +local ps = require("program.ps.ps") local keys, fatal = lwutil.keys, lwutil.fatal @@ -97,12 +99,30 @@ function run (raw_args) counter_name, pid = nil, pid end if opts.name then - -- without --reconfigurable + end + if opts.name then + -- Start by assuming it was run without --reconfigurable local programs = engine.enumerate_named_programs(opts.name) pid = programs[opts.name] if not pid then fatal(("Couldn't find process with name '%s'"):format(opts.name)) end + + -- Check if it was run with --reconfigurable + -- If it was, find the children, then find the pid of their parent. + -- Note that this approach will break as soon as there can be multiple + -- followers which need to have their statistics aggregated, as it will + -- only print the statistics for one child, not for all of them. + for _, name in ipairs(shm.children("/")) do + local p = tonumber(name) + local name = ps.appname_resolver(p) + if p and ps.is_worker(p) then + local leader_pid = tonumber(ps.get_leader_pid(p)) + -- If the precomputed by-name pid is the leader pid, set the pid + -- to be the follower's pid instead to get meaningful counters. + if leader_pid == pid then pid = p end + end + end end if not pid then fatal("No pid or name specified") end local instance_tree = top.select_snabb_instance(pid) diff --git a/src/program/ps/ps.lua b/src/program/ps/ps.lua index 0d4bfaa5d4..629fe9ba86 100644 --- a/src/program/ps/ps.lua +++ b/src/program/ps/ps.lua @@ -22,7 +22,7 @@ local function parse_args (args) if #args ~= 0 then usage(1) end end -local function appname_resolver() +function appname_resolver() local instances = {} for name, pid in pairs(app.enumerate_named_programs()) do instances[pid] = name @@ -30,7 +30,7 @@ local function appname_resolver() return function (pid) return instances[pid] end end -local function is_worker (pid) +function is_worker (pid) return shm.exists("/"..pid.."/group") end @@ -46,7 +46,7 @@ local function is_addressable (pid) return false end -local function get_leader_pid (pid) +function get_leader_pid (pid) local fq = shm.root.."/"..pid.."/group" local path = S.readlink(fq) return basename(dirname(path)) From 8946b8f8d2ef492c141f5f742cde85718934282b Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Thu, 12 Jan 2017 15:34:06 +0100 Subject: [PATCH 467/631] Changed tests to work without load --- src/program/lwaftr/query/selftest.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/program/lwaftr/query/selftest.sh b/src/program/lwaftr/query/selftest.sh index c550f9f9ed..48f1bca8d5 100755 --- a/src/program/lwaftr/query/selftest.sh +++ b/src/program/lwaftr/query/selftest.sh @@ -81,6 +81,5 @@ pid=$(get_lwaftr_instance) if [[ -n "$pid" ]]; then test_lwaftr_query $pid test_lwaftr_query $pid -l - test_lwaftr_query_filter $pid "memuse" - test_lwaftr_query_filter $pid "in-ipv4" + test_lwaftr_query_filter $pid "memuse-ipv" fi From c70c5a73d64e2b4ea4c67d8dc9ca4d5b8b60b4e1 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Thu, 12 Jan 2017 17:09:28 +0100 Subject: [PATCH 468/631] Allow querying follower by leader pid --- src/program/lwaftr/query/query.lua | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 43f5c75d5f..a1c2089a0f 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -93,10 +93,26 @@ local function print_counters (tree, filter) end end +-- Return the pid that was specified, unless it was a leader process, +-- in which case, return the follower pid that actually has useful counters. +local function pid_to_parent(pid) + local pid = tonumber(pid) + for _, name in ipairs(shm.children("/")) do + local p = tonumber(name) + if p and ps.is_worker(p) then + local leader_pid = tonumber(ps.get_leader_pid(p)) + -- If the precomputed by-name pid is the leader pid, set the pid + -- to be the follower's pid instead to get meaningful counters. + if leader_pid == pid then pid = p end + end + end + return pid +end + function run (raw_args) local opts, pid, counter_name = parse_args(raw_args) if tostring(pid) and not counter_name then - counter_name, pid = nil, pid + counter_name, pid = nil, pid_to_parent(pid) end if opts.name then end @@ -115,7 +131,6 @@ function run (raw_args) -- only print the statistics for one child, not for all of them. for _, name in ipairs(shm.children("/")) do local p = tonumber(name) - local name = ps.appname_resolver(p) if p and ps.is_worker(p) then local leader_pid = tonumber(ps.get_leader_pid(p)) -- If the precomputed by-name pid is the leader pid, set the pid @@ -124,7 +139,9 @@ function run (raw_args) end end end - if not pid then fatal("No pid or name specified") end + if not pid then + top.select_snabb_instance(pid) + end local instance_tree = top.select_snabb_instance(pid) print_counters(instance_tree, counter_name) end From 21f65855707751ad1dbe10626f7da09bf70a2eea Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 13 Jan 2017 11:16:46 +0000 Subject: [PATCH 469/631] lwaftr query selftest - Test reconfigurable and non-reconfigurable modes. - Test query --list-all. - Test by PID and by name. - Test by leader PID and follower PID. --- src/program/lwaftr/query/selftest.sh | 85 +------------------ src/program/lwaftr/query/tests/test_env.sh | 71 ++++++++++++++++ src/program/lwaftr/query/tests/test_query.sh | 38 +++++++++ .../query/tests/test_query_reconfigurable.sh | 45 ++++++++++ 4 files changed, 156 insertions(+), 83 deletions(-) create mode 100644 src/program/lwaftr/query/tests/test_env.sh create mode 100755 src/program/lwaftr/query/tests/test_query.sh create mode 100755 src/program/lwaftr/query/tests/test_query_reconfigurable.sh diff --git a/src/program/lwaftr/query/selftest.sh b/src/program/lwaftr/query/selftest.sh index 48f1bca8d5..0d884555d9 100755 --- a/src/program/lwaftr/query/selftest.sh +++ b/src/program/lwaftr/query/selftest.sh @@ -1,85 +1,4 @@ #!/usr/bin/env bash -SKIPPED_CODE=43 - -if [[ -z "$SNABB_PCI0" ]]; then - echo "SNABB_PCI0 not set" - exit $SKIPPED_CODE -fi - -if [[ -z "$SNABB_PCI1" ]]; then - echo "SNABB_PCI1 not set" - exit $SKIPPED_CODE -fi - -LWAFTR_CONF=./program/lwaftr/tests/data/no_icmp.conf - -function tmux_launch { - command="$2 2>&1 | tee $3" - if [ -z "$tmux_session" ]; then - tmux_session=test_env-$$ - tmux new-session -d -n "$1" -s $tmux_session "$command" - else - tmux new-window -a -d -n "$1" -t $tmux_session "$command" - fi -} - -function kill_lwaftr { - ps aux | grep $SNABB_PCI0 | awk '{print $2}' | xargs kill 2>/dev/null -} - -function cleanup { - kill_lwaftr - exit -} - -trap cleanup EXIT HUP INT QUIT TERM - -function get_lwaftr_instance { - pids=$(ps aux | grep $SNABB_PCI0 | awk '{print $2}') - for pid in ${pids[@]}; do - if [[ -d "/var/run/snabb/$pid/apps/lwaftr" ]]; then - echo $pid - fi - done -} - -function fatal { - local msg=$1 - echo "Error: $msg" - exit 1 -} - -function test_lwaftr_query { - local pid=$1 - # FIXME: Sometimes lwaftr query gets stalled. Add timeout. - local lineno=`timeout 1 ./snabb lwaftr query $pid | wc -l` - if [[ $lineno -gt 1 ]]; then - echo "Success: lwaftr query $pid" - else - fatal "lwaftr query $pid" - fi -} - -function test_lwaftr_query_filter { - local pid=$1 - local filter=$2 - local lineno=`timeout 1 ./snabb lwaftr query $pid $filter | wc -l` - if [[ $lineno -gt 1 ]]; then - echo "Success: lwaftr query $pid $filter" - else - fatal "lwaftr query $pid" - fi -} - -# Run lwAFTR. -tmux_launch "lwaftr" "./snabb lwaftr run --reconfigurable --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" -sleep 2 - -# Run tests. -pid=$(get_lwaftr_instance) -if [[ -n "$pid" ]]; then - test_lwaftr_query $pid - test_lwaftr_query $pid -l - test_lwaftr_query_filter $pid "memuse-ipv" -fi +./program/lwaftr/query/tests/test_query_reconfigurable.sh +./program/lwaftr/query/tests/test_query.sh diff --git a/src/program/lwaftr/query/tests/test_env.sh b/src/program/lwaftr/query/tests/test_env.sh new file mode 100644 index 0000000000..c4e4350fe7 --- /dev/null +++ b/src/program/lwaftr/query/tests/test_env.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +TEMP_FILE=$(mktemp) + +function tmux_launch { + command="$2 2>&1 | tee $3" + if [ -z "$tmux_session" ]; then + tmux_session=test_env-$$ + tmux new-session -d -n "$1" -s $tmux_session "$command" + else + tmux new-window -a -d -n "$1" -t $tmux_session "$command" + fi +} + +function kill_lwaftr { + ps aux | grep $SNABB_PCI0 | awk '{print $2}' | xargs kill 2>/dev/null +} + +function cleanup { + kill_lwaftr + rm -f $TEMP_FILE + exit +} + +function fatal { + local msg=$1 + echo "Error: $msg" + exit 1 +} + +function get_lwaftr_leader { + local pids=$(ps aux | grep "\-\-reconfigurable" | grep $SNABB_PCI0 | grep -v "grep" | awk '{print $2}') + for pid in ${pids[@]}; do + if [[ -d "/var/run/snabb/$pid" ]]; then + echo $pid + fi + done +} + +function get_lwaftr_follower { + local leaders=$(ps aux | grep "\-\-reconfigurable" | grep $SNABB_PCI0 | grep -v "grep" | awk '{print $2}') + for pid in $(ls /var/run/snabb); do + for leader in ${leaders[@]}; do + if [[ -L "/var/run/snabb/$pid/group" ]]; then + local target=$(ls -l /var/run/snabb/$pid/group | awk '{print $11}' | grep -oe "[0-9]\+") + if [[ "$leader" == "$target" ]]; then + echo $pid + fi + fi + done + done +} + +function get_lwaftr_instance { + local pids=$(ps aux | grep $SNABB_PCI0 | awk '{print $2}') + for pid in ${pids[@]}; do + if [[ -d "/var/run/snabb/$pid/apps/lwaftr" ]]; then + echo $pid + fi + done +} + +function test_lwaftr_query { + ./snabb lwaftr query $@ > $TEMP_FILE + local lineno=`cat $TEMP_FILE | wc -l` + if [[ $lineno -gt 1 ]]; then + echo "Success: lwaftr query $*" + else + fatal "lwaftr query $*" + fi +} diff --git a/src/program/lwaftr/query/tests/test_query.sh b/src/program/lwaftr/query/tests/test_query.sh new file mode 100755 index 0000000000..c31033f363 --- /dev/null +++ b/src/program/lwaftr/query/tests/test_query.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +SKIPPED_CODE=43 + +if [[ -z "$SNABB_PCI0" ]]; then + echo "SNABB_PCI0 not set" + exit $SKIPPED_CODE +fi + +if [[ -z "$SNABB_PCI1" ]]; then + echo "SNABB_PCI1 not set" + exit $SKIPPED_CODE +fi + +source ./program/lwaftr/query/tests/test_env.sh + +trap cleanup EXIT HUP INT QUIT TERM + +LWAFTR_CONF=./program/lwaftr/tests/data/no_icmp.conf +LWAFTR_NAME=lwaftr-$$ + +# Run lwAFTR. +tmux_launch "lwaftr" "./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" "lwaftr.log" +sleep 2 + +# Test query all. +test_lwaftr_query -l + +# Test query by pid. +pid=$(get_lwaftr_instance) +if [[ -n "$pid" ]]; then + test_lwaftr_query $pid + test_lwaftr_query $pid "memuse-ipv" +fi + +# Test query by name. +test_lwaftr_query "--name $LWAFTR_NAME" +test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" diff --git a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh new file mode 100755 index 0000000000..7065a5e706 --- /dev/null +++ b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +SKIPPED_CODE=43 + +if [[ -z "$SNABB_PCI0" ]]; then + echo "SNABB_PCI0 not set" + exit $SKIPPED_CODE +fi + +if [[ -z "$SNABB_PCI1" ]]; then + echo "SNABB_PCI1 not set" + exit $SKIPPED_CODE +fi + +source ./program/lwaftr/query/tests/test_env.sh + +trap cleanup EXIT HUP INT QUIT TERM + +LWAFTR_CONF=./program/lwaftr/tests/data/no_icmp.conf +LWAFTR_NAME=lwaftr-$$ + +# Run lwAFTR. +tmux_launch "lwaftr" "./snabb lwaftr run --reconfigurable --name $LWAFTR_NAME --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" "lwaftr.log" +sleep 2 + +# Test query all. +test_lwaftr_query -l + +# Test query by leader pid. +pid=$(get_lwaftr_leader) +if [[ -n "$pid" ]]; then + test_lwaftr_query $pid + test_lwaftr_query $pid "memuse-ipv" +fi + +# Test query by follower pid. +pid=$(get_lwaftr_follower) +if [[ -n "$pid" ]]; then + test_lwaftr_query $pid + test_lwaftr_query $pid "memuse-ipv" +fi + +# Test query by name. +test_lwaftr_query "--name $LWAFTR_NAME" +test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" From f4543c1485b29b7fbe94bdcd561ffe6ffbad2999 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 13 Jan 2017 13:07:25 +0100 Subject: [PATCH 470/631] Don't return bogus values when no pid specified --- src/program/lwaftr/query/query.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index a1c2089a0f..707e9615b6 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -96,6 +96,8 @@ end -- Return the pid that was specified, unless it was a leader process, -- in which case, return the follower pid that actually has useful counters. local function pid_to_parent(pid) + -- It's meaningless to get the parent of a nil 'pid'. + if not pid then return pid end local pid = tonumber(pid) for _, name in ipairs(shm.children("/")) do local p = tonumber(name) @@ -114,8 +116,6 @@ function run (raw_args) if tostring(pid) and not counter_name then counter_name, pid = nil, pid_to_parent(pid) end - if opts.name then - end if opts.name then -- Start by assuming it was run without --reconfigurable local programs = engine.enumerate_named_programs(opts.name) From 407d05112254cebe2f74af5c333b62a70a161b4d Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 13 Jan 2017 13:33:17 +0100 Subject: [PATCH 471/631] Get the right pid, regardless of counter filters --- src/program/lwaftr/query/query.lua | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 707e9615b6..7351db3754 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -113,9 +113,7 @@ end function run (raw_args) local opts, pid, counter_name = parse_args(raw_args) - if tostring(pid) and not counter_name then - counter_name, pid = nil, pid_to_parent(pid) - end + if tostring(pid) then pid = pid_to_parent(pid) end if opts.name then -- Start by assuming it was run without --reconfigurable local programs = engine.enumerate_named_programs(opts.name) From 6bf608486e83884fb50754e72cc20e973b845d11 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 13 Jan 2017 13:55:38 +0100 Subject: [PATCH 472/631] Make filtering work with by-name --- src/program/lwaftr/query/query.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 7351db3754..a6ce2afab5 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -113,8 +113,13 @@ end function run (raw_args) local opts, pid, counter_name = parse_args(raw_args) - if tostring(pid) then pid = pid_to_parent(pid) end + if not opts.name then + if tostring(pid) then pid = pid_to_parent(pid) end + end + if opts.name then + -- by-name: arguments are shifted by 1 and no pid is specified + counter_name = pid -- Start by assuming it was run without --reconfigurable local programs = engine.enumerate_named_programs(opts.name) pid = programs[opts.name] From 72a801ee1ab6a3de82371d98f8297c3edec0ebc6 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 13 Jan 2017 14:28:44 +0100 Subject: [PATCH 473/631] Test counter name that does not exist --- src/program/lwaftr/query/tests/test_env.sh | 10 ++++++++++ src/program/lwaftr/query/tests/test_query.sh | 2 ++ .../lwaftr/query/tests/test_query_reconfigurable.sh | 3 +++ 3 files changed, 15 insertions(+) diff --git a/src/program/lwaftr/query/tests/test_env.sh b/src/program/lwaftr/query/tests/test_env.sh index c4e4350fe7..6db4f66b1e 100644 --- a/src/program/lwaftr/query/tests/test_env.sh +++ b/src/program/lwaftr/query/tests/test_env.sh @@ -69,3 +69,13 @@ function test_lwaftr_query { fatal "lwaftr query $*" fi } + +function test_lwaftr_query_no_counters { + ./snabb lwaftr query $@ > $TEMP_FILE + local lineno=`cat $TEMP_FILE | wc -l` + if [[ $lineno -eq 1 ]]; then + echo "Success: lwaftr query $*" + else + fatal "lwaftr query $*" + fi +} diff --git a/src/program/lwaftr/query/tests/test_query.sh b/src/program/lwaftr/query/tests/test_query.sh index c31033f363..c3df4bc49a 100755 --- a/src/program/lwaftr/query/tests/test_query.sh +++ b/src/program/lwaftr/query/tests/test_query.sh @@ -31,8 +31,10 @@ pid=$(get_lwaftr_instance) if [[ -n "$pid" ]]; then test_lwaftr_query $pid test_lwaftr_query $pid "memuse-ipv" + test_lwaftr_query_no_counters $pid counter-never-exists-123 fi # Test query by name. test_lwaftr_query "--name $LWAFTR_NAME" test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" +test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" diff --git a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh index 7065a5e706..f28ed4d9e6 100755 --- a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh +++ b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh @@ -31,6 +31,7 @@ pid=$(get_lwaftr_leader) if [[ -n "$pid" ]]; then test_lwaftr_query $pid test_lwaftr_query $pid "memuse-ipv" + test_lwaftr_query_no_counters $pid counter-never-exists-123 fi # Test query by follower pid. @@ -38,8 +39,10 @@ pid=$(get_lwaftr_follower) if [[ -n "$pid" ]]; then test_lwaftr_query $pid test_lwaftr_query $pid "memuse-ipv" + test_lwaftr_query_no_counters $pid counter-never-exists-123 fi # Test query by name. test_lwaftr_query "--name $LWAFTR_NAME" test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" +test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" From b051cf9f870740d32f2548f9ec6a23e8e960a76c Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 13 Jan 2017 15:04:47 +0100 Subject: [PATCH 474/631] Addressed nits --- src/program/lwaftr/query/query.lua | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index a6ce2afab5..e4346d439d 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -112,14 +112,13 @@ local function pid_to_parent(pid) end function run (raw_args) - local opts, pid, counter_name = parse_args(raw_args) + local opts, arg1, arg2 = parse_args(raw_args) + local pid, counter_name if not opts.name then - if tostring(pid) then pid = pid_to_parent(pid) end - end - - if opts.name then - -- by-name: arguments are shifted by 1 and no pid is specified - counter_name = pid + if arg1 then pid = pid_to_parent(arg1) end + counter_name = arg2 -- This may be nil + else -- by-name: arguments are shifted by 1 and no pid is specified + counter_name = arg1 -- Start by assuming it was run without --reconfigurable local programs = engine.enumerate_named_programs(opts.name) pid = programs[opts.name] @@ -143,8 +142,10 @@ function run (raw_args) end end if not pid then + print("pid, arg1, arg2", pid, arg1, arg2) top.select_snabb_instance(pid) + -- The following is not reached when there are multiple instances. + fatal("Please manually specify a pid, or a name with -n name") end - local instance_tree = top.select_snabb_instance(pid) - print_counters(instance_tree, counter_name) + print_counters(pid, counter_name) end From b31d1db2bcb40dad7b3cf59dce42561cd5169d14 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 13 Jan 2017 15:16:06 +0100 Subject: [PATCH 475/631] removed print --- src/program/lwaftr/query/query.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index e4346d439d..0768ffc3f0 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -142,7 +142,6 @@ function run (raw_args) end end if not pid then - print("pid, arg1, arg2", pid, arg1, arg2) top.select_snabb_instance(pid) -- The following is not reached when there are multiple instances. fatal("Please manually specify a pid, or a name with -n name") From 2c36f5243ad37df8b5b61b1bb874312070fa6fbc Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 13 Jan 2017 14:46:43 +0000 Subject: [PATCH 476/631] Print NYI error for softwire-num-threshold --- src/apps/config/support/snabb-softwire-v1.lua | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index f1e5a52254..59f072ed92 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -221,7 +221,7 @@ local function ietf_softwire_translator () br_instance[br_instance_key_t({id=1})] = { tunnel_payload_mtu = native_config.softwire_config.internal_interface.mtu, tunnel_path_mru = native_config.softwire_config.external_interface.mtu, - -- FIXME: What it should map to? + -- FIXME: There's no equivalent of softwire-num-threshold in snabb-softwire-v1. softwire_num_threshold = 0xffffffff, binding_table = { binding_entry = ietf_binding_table_from_native( @@ -276,7 +276,7 @@ local function ietf_softwire_translator () 'br-instances', 'br-instance'} local bt_paths = {'binding-table', 'binding-entry'} - -- Handle special br attributes (tunnel-payload-mtu, tunnel-path-mru, softwire-num-threshold) + -- Handle special br attributes (tunnel-payload-mtu, tunnel-path-mru, softwire-num-threshold). if #path > #br_instance_paths then if path[#path].name == 'tunnel-payload-mtu' then return {{'set', {schema='snabb-softwire-v1', @@ -289,8 +289,7 @@ local function ietf_softwire_translator () config=tostring(arg)}}} end if path[#path].name == 'softwire-num-threshold' then - -- FIXME: Do nothing. - return {} + error('not yet implemented: softwire-num-threshold') end error('unrecognized leaf: '..path[#path].name) end From 884c938cd503de3287cccded734416f18209f094 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Mon, 16 Jan 2017 06:42:03 +0000 Subject: [PATCH 477/631] Fix `snabb config remove` on arrays --- src/apps/config/leader.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index f317d0bfaf..807cdbc403 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -319,17 +319,18 @@ local function path_remover_for_grammar(grammar, path) local head, tail = lib.dirname(path), lib.basename(path) local tail_path = path_mod.parse_path(tail) local tail_name, query = tail_path[1].name, tail_path[1].query - local getter, grammar = path_mod.resolver(grammar, head..'/'..tail_name) + local head_and_tail_name = head..'/'..tail_name + local getter, grammar = path_mod.resolver(grammar, head_and_tail_name) if grammar.type == 'array' then if grammar.ctype then -- It's an FFI array; have to create a fresh one, sadly. local idx = path_mod.prepare_array_lookup(query) - local setter = path_setter_for_grammar(top_grammar, head) + local setter = path_setter_for_grammar(top_grammar, head_and_tail_name) local elt_t = data.typeof(grammar.ctype) local array_t = ffi.typeof('$[?]', elt_t) return function(config) local cur = getter(config) - assert(i <= #cur) + assert(idx <= #cur) local new = array_t(#cur - 1) for i,elt in ipairs(cur) do if i < idx then new[i-1] = elt end From c6e58c15300df143af885c56889e2f72c170d201 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Mon, 16 Jan 2017 06:49:49 +0000 Subject: [PATCH 478/631] Fix typo in `snabb config` README --- src/program/config/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/config/README.md b/src/program/config/README.md index eea6915fff..fe07e3a9f8 100644 --- a/src/program/config/README.md +++ b/src/program/config/README.md @@ -232,7 +232,7 @@ Using `snabb config load` has the advantage that any configuration error has a corresponding source location. `snabb config` can also remove part of a configuration, but only on -configuration that corresponds to YANG schema `leaf` or `leaf-list` +configuration that corresponds to YANG schema `list` or `leaf-list` nodes: ``` From 5b4a6d0923c328e48e292c4391152dc484a90846 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Jan 2017 10:03:52 +0100 Subject: [PATCH 479/631] Add tests for snabb listen over unix socket --- .../lwaftr/tests/config/test-config-listen.sh | 74 +++++++++++++++++++ src/program/lwaftr/tests/config/tools.sh | 14 +++- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 src/program/lwaftr/tests/config/test-config-listen.sh diff --git a/src/program/lwaftr/tests/config/test-config-listen.sh b/src/program/lwaftr/tests/config/test-config-listen.sh new file mode 100644 index 0000000000..e61280bb8a --- /dev/null +++ b/src/program/lwaftr/tests/config/test-config-listen.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +## This checks it can listen, send a command and get a +## response. It only tests the socket method of communicating +## with the listen command due to the difficulties of testing +## interactive scripts. + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +# Verify we have the "nc" tool, used to communicate with sockets +# if we don't have it we just have to skip this test. +which nc &> /dev/null +if [[ $? -ne 0 ]]; then + echo "No 'nc' tool present, unable to run test." 1&>2 + exit 1 +fi + +# Load the tools to be able to test stuff. +BASEDIR="`pwd`" +cd "`dirname \"$0\"`" +source tools.sh +cd $BASEDIR + +# Come up with a name for the lwaftr +SNABB_NAME="`random_name`" + +# Start the bench command. +start_lwaftr_bench $SNABB_NAME + +# Start the listen command with a socket +SOCKET_PATH="/tmp/snabb-test-listen-sock-$SNABB_NAME" +./snabb config listen --socket "$SOCKET_PATH" "$SNABB_NAME" &> /dev/null & + +# It shouldn't take long but wait a short while for the socket +# to be created. +sleep 1 + +# Create input and output fifo's to communicate. +LISTEN_IN=$(mktemp -u) +LISTEN_OUT=$(mktemp -u) +mkfifo "$LISTEN_IN" +mkfifo "$LISTEN_OUT" + +# Start a communication with the listen program +(cat "$LISTEN_IN" | nc -U "$SOCKET_PATH" > "$LISTEN_OUT") & + +# Get the PID of nc so it can be easily stopped later +NC_PID=$! + +# Finally, lets send a get command. +GET_CMD="{ \"id\": \"0\", \"verb\": \"get\", \"path\": \"/routes/route[addr=1.2.3.4]/port\" }" +echo "$GET_CMD" > "$LISTEN_IN" + +# Sleep a short amount of time to let it respond, one second should be more than plenty +sleep 1 + +# Read the response from the listen command +GET_CMD_RESPONSE=$(cat "$LISTEN_OUT") + +# Check the response as best I can, I'll use python as it's common to most. +PARSED_GET_RESPONSE=$(echo $GET_CMD_RESPONSE | python -c " +import json, sys + +print(json.loads(sys.stdin.read(200))[\"status\"])" +) + +# Test the status is "ok" +assert_equal "$PARSED_GET_RESPONSE" "ok" + +# Finally end all the programs we've spawned. +stop_if_running "$NC_PID" +stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/tools.sh b/src/program/lwaftr/tests/config/tools.sh index b55dac446f..978baedde4 100755 --- a/src/program/lwaftr/tests/config/tools.sh +++ b/src/program/lwaftr/tests/config/tools.sh @@ -37,7 +37,7 @@ function assert_equal { # duration is set to prevent it running indefinitely. function start_lwaftr_bench { ./snabb lwaftr bench --reconfigurable --bench-file /dev/null --name "$1" \ - --duration 20 \ + --duration 30 \ program/lwaftr/tests/data/icmp_on_fail.conf \ program/lwaftr/tests/benchdata/ipv{4,6}-0550.pcap &> /dev/null & @@ -52,3 +52,15 @@ function stop_lwaftr_bench { # Wait until it's shutdown. wait &> /dev/null } + +function stop_if_running { + # Check if it's running, if not, job done. + kill -0 "$1" &> /dev/null + if [[ "$?" -ne 0 ]]; then + return + fi + + # It's running, lets try and close it nicely + kill -15 "$1" + wait &> /dev/null +} From 97eeef17fcf63966aef34117abd49db8401338ee Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Jan 2017 10:15:11 +0100 Subject: [PATCH 480/631] Add snabb config listen test to selftest.sh --- src/program/lwaftr/tests/config/selftest.sh | 1 + src/program/lwaftr/tests/config/test-config-listen.sh | 0 2 files changed, 1 insertion(+) mode change 100644 => 100755 src/program/lwaftr/tests/config/test-config-listen.sh diff --git a/src/program/lwaftr/tests/config/selftest.sh b/src/program/lwaftr/tests/config/selftest.sh index c9ffba7066..817410f78c 100755 --- a/src/program/lwaftr/tests/config/selftest.sh +++ b/src/program/lwaftr/tests/config/selftest.sh @@ -7,3 +7,4 @@ set -o errexit ./program/lwaftr/tests/config/test-config-add.sh ./program/lwaftr/tests/config/test-config-remove.sh ./program/lwaftr/tests/config/test-config-get-state.sh +./program/lwaftr/tests/config/test-config-listen.sh diff --git a/src/program/lwaftr/tests/config/test-config-listen.sh b/src/program/lwaftr/tests/config/test-config-listen.sh old mode 100644 new mode 100755 From 051545151c6eac6b67ad87c554c20170f7634156 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Jan 2017 11:06:42 +0100 Subject: [PATCH 481/631] test-config-listen.sh: use skipped test exit code --- src/program/lwaftr/tests/config/test-config-listen.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/config/test-config-listen.sh b/src/program/lwaftr/tests/config/test-config-listen.sh index e61280bb8a..29de3c9d60 100755 --- a/src/program/lwaftr/tests/config/test-config-listen.sh +++ b/src/program/lwaftr/tests/config/test-config-listen.sh @@ -3,6 +3,7 @@ ## response. It only tests the socket method of communicating ## with the listen command due to the difficulties of testing ## interactive scripts. +SKIPPED_CODE=43 if [[ $EUID -ne 0 ]]; then echo "This script must be run as root" 1>&2 @@ -14,7 +15,7 @@ fi which nc &> /dev/null if [[ $? -ne 0 ]]; then echo "No 'nc' tool present, unable to run test." 1&>2 - exit 1 + exit $SKIPPED_CODE fi # Load the tools to be able to test stuff. From c0866eb46960ded6e4abed5e13ef8dda2c3d6696 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 17 Jan 2017 11:36:33 +0100 Subject: [PATCH 482/631] Re-trigger unit tests From 8c401dc30eebfd033f2b44b26ab23512d7183a3a Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 17 Jan 2017 12:27:40 +0000 Subject: [PATCH 483/631] Respond to ping6 packets in on-a-stick mode Adapt function requires_splitter to new configuration naming. --- src/program/lwaftr/run/run.lua | 13 ++++++++----- src/program/lwaftr/setup.lua | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index bbf681730e..8b6cb78ee2 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -121,12 +121,15 @@ function parse_args(args) end end --- Requires a V4V6 splitter iff: --- Always when running in on-a-stick mode, except if v4_vlan_tag != v6_vlan_tag. +-- Requires a V4V6 splitter if running in on-a-stick mode and VLAN tag values +-- are the same for the internal and external interfaces. local function requires_splitter (opts, conf) - if not opts["on-a-stick"] then return false end - if not conf.vlan_tagging then return true end - return conf.v4_vlan_tag == conf.v6_vlan_tag + if opts["on-a-stick"] then + local internal_interface = conf.softwire_config.internal_interface + local external_interface = conf.softwire_config.external_interface + return internal_interface.vlan_tag == external_interface.vlan_tag + end + return false end function run(args) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index ca0cda0639..8fce523fd3 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -215,7 +215,7 @@ function load_on_a_stick(c, conf, args) pciaddr = pciaddr, vmdq=internal_interface.vlan_tag, vlan=internal_interface.vlan_tag, - macaddr = ethernet:ntop(conf.internal_interface.mac)}) + macaddr = ethernet:ntop(internal_interface.mac)}) link_source(c, v4_nic_name..'.tx', v6_nic_name..'.tx') link_sink(c, v4_nic_name..'.rx', v6_nic_name..'.rx') From 12e3e3aa3564797294124e834e5a4b111fcbd4ed Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 17 Jan 2017 12:29:27 +0000 Subject: [PATCH 484/631] Add test ping lwAFTR's V4 and V6 interfaces from a Linux kernel interface lwAFTR runs in on-a-stick mode. --- src/program/lwaftr/tests/hw/selftest.sh | 3 + .../lwaftr/tests/hw/test_ping_on_a_stick.sh | 112 ++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100755 src/program/lwaftr/tests/hw/selftest.sh create mode 100755 src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh diff --git a/src/program/lwaftr/tests/hw/selftest.sh b/src/program/lwaftr/tests/hw/selftest.sh new file mode 100755 index 0000000000..c9980d435d --- /dev/null +++ b/src/program/lwaftr/tests/hw/selftest.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./program/lwaftr/tests/hw/test_ping_on_a_stick.sh diff --git a/src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh b/src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh new file mode 100755 index 0000000000..fee5adf44d --- /dev/null +++ b/src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +SKIPPED_CODE=43 + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +if [[ -z "$SNABB_PCI0" ]]; then + echo "SNABB_PCI0 not defined" + exit $SKIPPED_CODE +fi + +if [[ -z "$SNABB_PCI1" ]]; then + echo "SNABB_PCI1 not defined" + exit $SKIPPED_CODE +fi + +function fatal { + local msg=$1 + echo "Error: $msg" + exit 1 +} + +function iface_by_pciaddr { + local pciaddr=$1 + echo $(lshw -c network -businfo | grep "pci@$pciaddr" | awk '{print $2}') +} + +function bind_card { + local pciaddr=$1 + + # Check is bound. + if [[ -L "/sys/bus/pci/drivers/ixgbe/$pciaddr" ]]; then + echo $(iface_by_pciaddr $pciaddr) + fi + + # Bind card and return iface name. + echo $pciaddr | sudo tee /sys/bus/pci/drivers/ixgbe/bind &> /dev/null + if [[ $? -eq 0 ]]; then + iface=$(ls /sys/bus/pci/devices/$pciaddr/net) + echo $iface + fi +} + +function test_ping_to_internal_interface { + local out=$(ping6 -c 1 -I "$IFACE.v6" "$INTERNAL_IP") + local count=$(echo "$out" | grep -o -c " 0% packet loss") + if [[ $count -eq 1 ]]; then + echo "Success: Ping to internal interface" + else + fatal "Couldn't ping to internal interface" + fi +} + +function test_ping_to_external_interface { + local out=$(ping -c 1 -I "$IFACE.v4" "$EXTERNAL_IP") + local count=$(echo "$out" | grep -o -c " 0% packet loss") + if [[ $count -eq 1 ]]; then + echo "Success: Ping to external interface" + else + fatal "Couldn't ping to internal interface" + fi + +} + +function cleanup { + sudo tmux kill-session -t $lwaftr_session + local pids=$(ps aux | grep $SNABB_PCI0 | grep -v grep | awk '{print $2}') + for pid in ${pids[@]}; do + kill $pid + done +} + +trap cleanup EXIT HUP INT QUIT TERM + +LWAFTR_CONF=lwaftr-migrated.conf +EXTERNAL_IP=10.0.1.1 +INTERNAL_IP=fe80::100 +IPV4_ADDRESS=10.0.1.2/24 +IPV6_ADDRESS=fe80::101/64 +VLAN_V4_TAG=164 +VLAN_V6_TAG=125 + +# Bind SNABB_PCI1 to kernel. +IFACE=$(bind_card $SNABB_PCI1) +if [[ -z "$IFACE" ]]; then + fatal "Couldn't bind card $SNABB_PCI1" +else + ip li set up dev $IFACE + sleep 1 +fi + +# Run lwAFTR in on-a-stick mode. +lwaftr_session=lwaftr-session-$$ +tmux new-session -d -n "lwaftr" -s $lwaftr_session "sudo ./snabb lwaftr run --conf $LWAFTR_CONF --on-a-stick $SNABB_PCI0" | tee lwaftr.log + +# Create VLAN V4 interface. +ip li delete "$IFACE.v4" &> /dev/null +ip link add link $IFACE name "$IFACE.v4" type vlan id $VLAN_V4_TAG +ip addr add $IPV4_ADDRESS dev "$IFACE.v4" +sleep 3 + +# Create VLAN V6 interface. +ip li delete "$IFACE.v6" &> /dev/null +ip link add link $IFACE name "$IFACE.v6" type vlan id $VLAN_V6_TAG +ip addr add $IPV6_ADDRESS dev "$IFACE.v6" +sleep 3 + +test_ping_to_internal_interface +test_ping_to_external_interface From 18d635c8c9b8eec2a94280a39c1f899b428e2d50 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 17 Jan 2017 13:19:14 +0000 Subject: [PATCH 485/631] Add configuration file --- .../lwaftr/tests/data/lwaftr-vlan.conf | 49 +++++++++++++++++++ .../lwaftr/tests/hw/test_ping_on_a_stick.sh | 4 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 src/program/lwaftr/tests/data/lwaftr-vlan.conf diff --git a/src/program/lwaftr/tests/data/lwaftr-vlan.conf b/src/program/lwaftr/tests/data/lwaftr-vlan.conf new file mode 100644 index 0000000000..3ae15ef56d --- /dev/null +++ b/src/program/lwaftr/tests/data/lwaftr-vlan.conf @@ -0,0 +1,49 @@ +softwire-config { + binding-table { + br-address fc00::100; + psid-map { + addr 193.5.1.100; + end-addr 193.5.63.101; + psid-length 6; + shift 10; + } + softwire { + ipv4 193.5.17.1; + psid 44; + b4-ipv6 fc00:1:2:3:4:5:3:d84c; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip 10.0.1.1; + mac 02:aa:aa:aa:aa:aa; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 164; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + generate-icmp-errors false; + ip fe80::100; + mac 02:aa:aa:aa:aa:aa; + mtu 9500; + next-hop { + mac 02:99:99:99:99:99; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 125; + } +} diff --git a/src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh b/src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh index fee5adf44d..3faf2b5917 100755 --- a/src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh +++ b/src/program/lwaftr/tests/hw/test_ping_on_a_stick.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# set -x + SKIPPED_CODE=43 if [[ $EUID -ne 0 ]]; then @@ -75,7 +77,7 @@ function cleanup { trap cleanup EXIT HUP INT QUIT TERM -LWAFTR_CONF=lwaftr-migrated.conf +LWAFTR_CONF=program/lwaftr/tests/data/lwaftr-vlan.conf EXTERNAL_IP=10.0.1.1 INTERNAL_IP=fe80::100 IPV4_ADDRESS=10.0.1.2/24 From 9d90e5d205da5ca90bad0230ee54d35a3e2d1ffe Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 17 Jan 2017 15:38:41 +0000 Subject: [PATCH 486/631] Skip hw ping to lwAFTR test in CI The program/lwaftr/tests/hw/test_ping_on_a_stick.sh test cannot be run inside a container for several reasons: - The test contains commands, such as iproute2 or lshw, not supported in the default image. - The test binds a NIC to the Linux kernel by PCI address. This enables the NIC interface. However, when run inside the container, the card gets rebound to the kernel, but the interface doesn't appear inside the container but the host. Without an addressable kernel-managed interface, it's not possible to run the test. --- src/program/lwaftr/tests/hw/selftest.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100755 src/program/lwaftr/tests/hw/selftest.sh diff --git a/src/program/lwaftr/tests/hw/selftest.sh b/src/program/lwaftr/tests/hw/selftest.sh deleted file mode 100755 index c9980d435d..0000000000 --- a/src/program/lwaftr/tests/hw/selftest.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -./program/lwaftr/tests/hw/test_ping_on_a_stick.sh From c28168225d5b59cf231a7f5a2ab42f21579f8eef Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Wed, 7 Dec 2016 23:39:23 +0100 Subject: [PATCH 487/631] Property-based test: lwaftr run does not crash with get This adds a first property based test. The property is that the lwaftr does not crash with a 'snabb config get' command. --- .../lwaftr/tests/propbased/genyang.lua | 12 ++ .../lwaftr/tests/propbased/prop_nocrash.lua | 43 ++++++ .../lwaftr/tests/propbased/selftest.sh | 9 ++ src/program/quickcheck/quickcheck.lua | 123 ++++++++++++++++++ src/program/quickcheck/utils.lua | 116 +++++++++++++++++ 5 files changed, 303 insertions(+) create mode 100644 src/program/lwaftr/tests/propbased/genyang.lua create mode 100644 src/program/lwaftr/tests/propbased/prop_nocrash.lua create mode 100755 src/program/lwaftr/tests/propbased/selftest.sh create mode 100644 src/program/quickcheck/quickcheck.lua create mode 100644 src/program/quickcheck/utils.lua diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua new file mode 100644 index 0000000000..25e29cf777 --- /dev/null +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -0,0 +1,12 @@ +module(..., package.seeall) + +--local S = require("syscall") +--local snabb_cmd = ("/proc/%d/exe"):format(S.getpid()) + +function generate_yang(pid) + return string.format("./snabb config get %s /", pid) +end + +function run_yang(yang_cmd) + return io.popen(yang_cmd):read("*a") +end diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua new file mode 100644 index 0000000000..136a3f9f9e --- /dev/null +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -0,0 +1,43 @@ +#!/usr/bin/env luajit +module(..., package.seeall) + +local genyang = require("program.lwaftr.tests.propbased.genyang") +local S = require("syscall") +local run_pid +local current_cmd + +function property() + current_cmd = genyang.generate_yang(run_pid) + local results = (genyang.run_yang(current_cmd)) + if string.match("Could not connect to config leader socket on Snabb instance", + results) then + print("Launching snabb run failed, or we've crashed it!") + return false + end +end + +function print_extra_information() + print("The command was:", current_cmd) +end + +function handle_prop_args(prop_args) + if #prop_args ~= 1 then + print("Usage: snabb quickcheck prop_nocrash PCI_ADDR") + os.exit(1) + end + + -- TODO: validate the address + local pci_addr = prop_args[1] + + local pid = S.fork() + if pid == 0 then + local cmdline = {"snabb", "lwaftr", "run", "-D", "2", "--conf", + "program/lwaftr/tests/data/icmp_on_fail.conf", "--reconfigurable", + "--on-a-stick", pci_addr} + -- FIXME: preserve the environment + S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, {}) + else + run_pid = pid + S.sleep(0.1) + end +end diff --git a/src/program/lwaftr/tests/propbased/selftest.sh b/src/program/lwaftr/tests/propbased/selftest.sh new file mode 100755 index 0000000000..1c1f6ce185 --- /dev/null +++ b/src/program/lwaftr/tests/propbased/selftest.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +SKIPPED_CODE=43 + +if [ -z $SNABB_PCI0 0 ]; then exit SKIPPED_CODE; fi + +./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash $SNABB_PCI0 diff --git a/src/program/quickcheck/quickcheck.lua b/src/program/quickcheck/quickcheck.lua new file mode 100644 index 0000000000..cd6972269f --- /dev/null +++ b/src/program/quickcheck/quickcheck.lua @@ -0,0 +1,123 @@ +module(...,package.seeall) + +local utils = require('program.quickcheck.utils') + +local program_name = 'snabb quickcheck' + +local seed, iterations, prop_name, prop_args, prop, prop_info + +-- Due to limitations of Lua 5.1, finding if a command failed is convoluted. +local function find_gitrev() + local fd = io.popen('git rev-parse HEAD 2>/dev/null ; echo -n "$?"') + local cmdout = fd:read("*all") + fd:close() -- Always true in 5.1, with Lua or LuaJIT + local _, _, git_ret = cmdout:find("(%d+)$") + git_ret = tonumber(git_ret) + if git_ret ~= 0 then -- Probably not in a git repo + return nil + else + local _, _, sha1 = cmdout:find("(%x+)") + return sha1 + end +end + +local function print_gitrev_if_available() + local rev = find_gitrev() + if rev then print(("Git revision %s"):format(rev)) end +end + +local function rerun_usage(i) + print(("Rerun as: %s --seed=%s --iterations=%s %s %s"): + format(program_name, seed, i + 1, + prop_name, table.concat(prop_args, " "))) +end + +function initialize(options) + seed, iterations, prop_name, prop_args = + options.seed, options.iterations, options.prop_name, options.prop_args + + if not seed then + seed = math.floor(utils.gmtime() * 1e6) % 10^9 + print("Using time as seed: "..seed) + end + math.randomseed(assert(tonumber(seed))) + + if not iterations then iterations = 100 end + + if not prop_name then + error("No property name specified") + end + + prop = require(prop_name) + if prop.handle_prop_args then + prop_info = prop.handle_prop_args(prop_args) + else + assert(#prop_args == 0, + "Property does not take options "..prop_name) + prop_info = nil + end +end + +function initialize_from_command_line(args) + local options = {} + while #args >= 1 and args[1]:match("^%-%-") do + local arg, _, val = table.remove(args, 1):match("^%-%-([^=]*)(=(.*))$") + assert(arg) + if arg == 'seed' then options.seed = assert(tonumber(val)) + elseif arg == 'iterations' then options.iterations = assert(tonumber(val)) + else error("Unknown argument: " .. arg) end + end + if #args < 1 then + print("Usage: " .. + program_name .. + " [--seed=SEED]" .. + " [--iterations=ITERATIONS]" .. + " property_file [property_specific_args]") + os.exit(1) + end + options.prop_name = table.remove(args, 1) + options.prop_args = args + initialize(options) +end + +function run(args) + initialize_from_command_line(args) + if not prop then + error("Call initialize() or initialize_from_command_line() first") + end + + for i = 1,iterations do + -- Wrap property and its arguments in a 0-arity function for xpcall + local wrap_prop = function() return prop.property(prop_info) end + local propgen_ok, expected, got = xpcall(wrap_prop, debug.traceback) + if not propgen_ok then + print(("Crashed generating properties on run %s."):format(i)) + if prop.print_extra_information then + print("Attempting to print extra information; it may be wrong.") + if not pcall(prop.print_extra_information) + then print("Something went wrong printing extra info.") + end + end + print("Traceback (this is reliable):") + print(expected) -- This is an error code and traceback in this case + rerun_usage(i) + os.exit(1) + end + if not utils.equals(expected, got) then + print_gitrev_if_available() + print("The property was falsified.") + -- If the property file has extra info available, show it + if prop.print_extra_information then + prop.print_extra_information() + else + print('Expected:') + utils.pp(expected) + print('Got:') + utils.pp(got) + end + rerun_usage(i) + os.exit(1) + end + end + print(iterations.." iterations succeeded.") +end diff --git a/src/program/quickcheck/utils.lua b/src/program/quickcheck/utils.lua new file mode 100644 index 0000000000..69235d4ca3 --- /dev/null +++ b/src/program/quickcheck/utils.lua @@ -0,0 +1,116 @@ +module(...,package.seeall) + +local S = require("syscall") + +function gmtime() + local tv = S.gettimeofday() + local secs = tonumber(tv.tv_sec) + secs = secs + tonumber(tv.tv_usec) * 1e-6 + return secs +end + +function concat(a, b) + local ret = {} + for _, v in ipairs(a) do table.insert(ret, v) end + for _, v in ipairs(b) do table.insert(ret, v) end + return ret +end + +function equals(expected, actual) + if type(expected) ~= type(actual) then return false end + if type(expected) == 'table' then + for k, v in pairs(expected) do + if not equals(v, actual[k]) then return false end + end + for k, _ in pairs(actual) do + if expected[k] == nil then return false end + end + return true + else + return expected == actual + end +end + +function is_array(x) + if type(x) ~= 'table' then return false end + if #x == 0 then return false end + for k,v in pairs(x) do + if type(k) ~= 'number' then return false end + -- Restrict to unsigned 32-bit integer keys. + if k < 0 or k >= 2^32 then return false end + -- Array indices are integers. + if k - math.floor(k) ~= 0 then return false end + -- Negative zero is not a valid array index. + if 1 / k < 0 then return false end + end + return true +end + +function pp(expr, indent, suffix) + indent = indent or '' + suffix = suffix or '' + if type(expr) == 'number' then + print(indent..expr..suffix) + elseif type(expr) == 'string' then + print(indent..'"'..expr..'"'..suffix) + elseif type(expr) == 'boolean' then + print(indent..(expr and 'true' or 'false')..suffix) + elseif is_array(expr) then + assert(#expr > 0) + if #expr == 1 then + if type(expr[1]) == 'table' then + print(indent..'{') + pp(expr[1], indent..' ', ' }'..suffix) + else + print(indent..'{ "'..expr[1]..'" }'..suffix) + end + else + if type(expr[1]) == 'table' then + print(indent..'{') + pp(expr[1], indent..' ', ',') + else + print(indent..'{ "'..expr[1]..'",') + end + indent = indent..' ' + for i=2,#expr-1 do pp(expr[i], indent, ',') end + pp(expr[#expr], indent, ' }'..suffix) + end + elseif type(expr) == 'table' then + if #expr == 0 then + print(indent .. '{}') + else + error('unimplemented') + end + else + error("unsupported type "..type(expr)) + end + return expr +end + +function assert_equals(expected, actual) + if not equals(expected, actual) then + pp(expected) + pp(actual) + error('not equal') + end +end + +function choose(choices) + local idx = math.random(#choices) + return choices[idx] +end + +function choose_with_index(choices) + local idx = math.random(#choices) + return choices[idx], idx +end + +function selftest () + print("selftest: pf.utils") + local tab = { 1, 2, 3 } + assert_equals({ 1, 2, 3, 1, 2, 3 }, concat(tab, tab)) + local gu1 = gmtime() + local gu2 = gmtime() + assert(gu1, gu2) + print("OK") +end From 4ad0c2ef305f94635dd82f5ab4fa7972709acf3a Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 7 Dec 2016 15:58:14 -0800 Subject: [PATCH 488/631] Minor fix to selftest.sh --- src/program/lwaftr/tests/propbased/selftest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/selftest.sh b/src/program/lwaftr/tests/propbased/selftest.sh index 1c1f6ce185..b14c0aa018 100755 --- a/src/program/lwaftr/tests/propbased/selftest.sh +++ b/src/program/lwaftr/tests/propbased/selftest.sh @@ -4,6 +4,6 @@ set -e SKIPPED_CODE=43 -if [ -z $SNABB_PCI0 0 ]; then exit SKIPPED_CODE; fi +if [ -z $SNABB_PCI0 ]; then exit SKIPPED_CODE; fi ./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash $SNABB_PCI0 From 88e6d0250b4f732be0552cddb1df9d207e3cfb53 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Thu, 8 Dec 2016 09:39:50 +0100 Subject: [PATCH 489/631] Close popens --- src/program/lwaftr/tests/propbased/genyang.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 25e29cf777..98fe690688 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -8,5 +8,8 @@ function generate_yang(pid) end function run_yang(yang_cmd) - return io.popen(yang_cmd):read("*a") + local f = io.popen(yang_cmd) + local result = f:read("*a") + f:close() + return result end From c7f669c19309bd0d313a1f52cb47dcded30ede58 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Thu, 8 Dec 2016 12:06:42 +0100 Subject: [PATCH 490/631] Add $ to exit code --- src/program/lwaftr/tests/propbased/selftest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/selftest.sh b/src/program/lwaftr/tests/propbased/selftest.sh index b14c0aa018..8bf81d7c42 100755 --- a/src/program/lwaftr/tests/propbased/selftest.sh +++ b/src/program/lwaftr/tests/propbased/selftest.sh @@ -4,6 +4,6 @@ set -e SKIPPED_CODE=43 -if [ -z $SNABB_PCI0 ]; then exit SKIPPED_CODE; fi +if [ -z $SNABB_PCI0 ]; then exit $SKIPPED_CODE; fi ./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash $SNABB_PCI0 From a28953b937d282c1d6c677d436e878cdc9b4b904 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 10:25:17 +0100 Subject: [PATCH 491/631] Run property-check lwaftr longer --- src/program/lwaftr/tests/propbased/prop_nocrash.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index 136a3f9f9e..edc013c094 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -31,7 +31,7 @@ function handle_prop_args(prop_args) local pid = S.fork() if pid == 0 then - local cmdline = {"snabb", "lwaftr", "run", "-D", "2", "--conf", + local cmdline = {"snabb", "lwaftr", "run", "-D", "20", "--conf", "program/lwaftr/tests/data/icmp_on_fail.conf", "--reconfigurable", "--on-a-stick", pci_addr} -- FIXME: preserve the environment From 4d6b1570f33074d669ce5f1f7291945779eca9bb Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Fri, 9 Dec 2016 10:26:28 +0100 Subject: [PATCH 492/631] Sleep for longer before starting test --- src/program/lwaftr/tests/propbased/prop_nocrash.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index edc013c094..dc9ae206cb 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -38,6 +38,6 @@ function handle_prop_args(prop_args) S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, {}) else run_pid = pid - S.sleep(0.1) + S.sleep(1) end end From 74bdc7403ab30c8f2a216eb7b928dc029b69237b Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 15:49:26 -0800 Subject: [PATCH 493/631] Generate xpath queries for `snabb config get` test --- .../lwaftr/tests/propbased/genyang.lua | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 98fe690688..4b7af76fde 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -3,8 +3,12 @@ module(..., package.seeall) --local S = require("syscall") --local snabb_cmd = ("/proc/%d/exe"):format(S.getpid()) +local schema = require("lib.yang.schema") +local softwire_schema = schema.load_schema_by_name("snabb-softwire-v1") + function generate_yang(pid) - return string.format("./snabb config get %s /", pid) + local query = generate_xpath(softwire_schema) + return string.format("./snabb config get %s \"%s\"", pid, query) end function run_yang(yang_cmd) @@ -13,3 +17,90 @@ function run_yang(yang_cmd) f:close() return result end + +-- choose an element of an array randomly +local function choose(choices) + local idx = math.random(#choices) + return choices[idx] +end + +-- choose from unbounded array indices, decreasing likelihood +local function choose_pos() + local r = math.random() + + local function flip(next) + local r = math.random() + if r < 0.5 then + return next + else + return flip(next + 1) + end + end + + -- evenly weight first five indices + if r < 0.5 then + return choose({1, 2, 3, 4, 5}) + else + return flip(6) + end +end + +-- from a config schema, generate an xpath query string +-- this code is patterned off of the visitor used in lib.yang.data +function generate_xpath(schema) + local path = "" + local handlers = {} + + local function visit(node) + local handler = handlers[node.kind] + if handler then handler(node) end + end + local function visit_body(node) + local ids = {} + for id, node in pairs(node.body) do + -- only choose nodes that are used in configs + if node.config ~= false then + table.insert(ids, id) + end + end + + local id = choose(ids) + if id then + visit(node.body[id]) + end + end + function handlers.container(node) + path = path .. "/" .. node.id + + -- don't always go into containers, since we need to test + -- fetching all sub-items too + if math.random() < 0.9 then + visit_body(node) + end + end + handlers['leaf-list'] = function(node) + local selector = string.format("[position()=%d]", choose_pos()) + path = path .. "/" .. node.id .. selector + end + function handlers.list(node) + -- TODO: this should generate selector-based lookups by using + -- the key type as a generation source, but for now do + -- the simple thing + path = path .. "/" .. node.id + end + function handlers.leaf(node) + path = path .. "/" .. node.id + end + + visit_body(schema) + + return path +end + +function selftest() + local data = require("lib.yang.data") + local path = require("lib.yang.path") + local grammar = data.data_grammar_from_schema(softwire_schema) + + path.convert_path(grammar, generate_xpath(softwire_schema)) +end From a0c779e02b3b3a0fdcf511ac226980b2153a264e Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 17:13:26 -0800 Subject: [PATCH 494/631] Add test ensuring two config runs yield the same result --- .../lwaftr/tests/propbased/prop_sameval.lua | 54 +++++++++++++++++++ .../lwaftr/tests/propbased/selftest.sh | 1 + 2 files changed, 55 insertions(+) create mode 100644 src/program/lwaftr/tests/propbased/prop_sameval.lua diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua new file mode 100644 index 0000000000..27dace611f --- /dev/null +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -0,0 +1,54 @@ +#!/usr/bin/env luajit +module(..., package.seeall) + +-- Make sure running a snabb config get twice results in the +-- same values getting returned + +local genyang = require("program.lwaftr.tests.propbased.genyang") +local S = require("syscall") +local run_pid +local current_cmd + +function property() + current_cmd = genyang.generate_yang(run_pid) + local results = (genyang.run_yang(current_cmd)) + local results2 = (genyang.run_yang(current_cmd)) + if string.match("Could not connect to config leader socket on Snabb instance", + results) or + string.match("Could not connect to config leader socket on Snabb instance", + results2) then + print("Launching snabb run failed, or we've crashed it!") + return false + end + + if results ~= results2 then + print("Running the same config command twice produced different outputs") + return false + end +end + +function print_extra_information() + print("The command was:", current_cmd) +end + +function handle_prop_args(prop_args) + if #prop_args ~= 1 then + print("Usage: snabb quickcheck prop_sameval PCI_ADDR") + os.exit(1) + end + + -- TODO: validate the address + local pci_addr = prop_args[1] + + local pid = S.fork() + if pid == 0 then + local cmdline = {"snabb", "lwaftr", "run", "-D", "20", "--conf", + "program/lwaftr/tests/data/icmp_on_fail.conf", "--reconfigurable", + "--on-a-stick", pci_addr} + -- FIXME: preserve the environment + S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, {}) + else + run_pid = pid + S.sleep(0.1) + end +end diff --git a/src/program/lwaftr/tests/propbased/selftest.sh b/src/program/lwaftr/tests/propbased/selftest.sh index 8bf81d7c42..d44ea32eba 100755 --- a/src/program/lwaftr/tests/propbased/selftest.sh +++ b/src/program/lwaftr/tests/propbased/selftest.sh @@ -7,3 +7,4 @@ SKIPPED_CODE=43 if [ -z $SNABB_PCI0 ]; then exit $SKIPPED_CODE; fi ./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash $SNABB_PCI0 +./snabb quickcheck program.lwaftr.tests.propbased.prop_sameval $SNABB_PCI0 From 56f53d81c3af9a77e7180662e747a2185f5f8e75 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 17:27:25 -0800 Subject: [PATCH 495/631] Add cleanup code to shutdown lwaftr process for test --- src/program/lwaftr/tests/propbased/prop_nocrash.lua | 4 ++++ src/program/lwaftr/tests/propbased/prop_sameval.lua | 4 ++++ src/program/quickcheck/quickcheck.lua | 2 ++ 3 files changed, 10 insertions(+) diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index dc9ae206cb..2b42abcc49 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -41,3 +41,7 @@ function handle_prop_args(prop_args) S.sleep(1) end end + +function cleanup() + S.kill(run_pid, "TERM") +end diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index 27dace611f..93da4ebaef 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -52,3 +52,7 @@ function handle_prop_args(prop_args) S.sleep(0.1) end end + +function cleanup() + S.kill(run_pid, "TERM") +end diff --git a/src/program/quickcheck/quickcheck.lua b/src/program/quickcheck/quickcheck.lua index cd6972269f..3dc7ba40ed 100644 --- a/src/program/quickcheck/quickcheck.lua +++ b/src/program/quickcheck/quickcheck.lua @@ -120,4 +120,6 @@ function run(args) end end print(iterations.." iterations succeeded.") + + if prop.cleanup then prop.cleanup() end end From 0002d5c9f261d7dfd513a77a116e4d99488e955f Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 20:15:46 -0800 Subject: [PATCH 496/631] Produce "/" path instead of "" in base case --- src/program/lwaftr/tests/propbased/genyang.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 4b7af76fde..f7aeb3ffe3 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -92,7 +92,10 @@ function generate_xpath(schema) path = path .. "/" .. node.id end - visit_body(schema) + -- just produce "/" on rare occasions + if math.random() > 0.01 then + visit_body(schema) + end return path end From 69146d2cf4ce7bfcbf639d5466e50746de41f19f Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 20:36:22 -0800 Subject: [PATCH 497/631] Abstract out the common prop testing code --- src/program/lwaftr/tests/propbased/common.lua | 37 +++++++++++++++++++ .../lwaftr/tests/propbased/prop_nocrash.lua | 33 +++-------------- .../lwaftr/tests/propbased/prop_sameval.lua | 33 +++-------------- 3 files changed, 49 insertions(+), 54 deletions(-) create mode 100644 src/program/lwaftr/tests/propbased/common.lua diff --git a/src/program/lwaftr/tests/propbased/common.lua b/src/program/lwaftr/tests/propbased/common.lua new file mode 100644 index 0000000000..733e3e1ff2 --- /dev/null +++ b/src/program/lwaftr/tests/propbased/common.lua @@ -0,0 +1,37 @@ +module(..., package.seeall) + +-- common definitions for property-based tests for snabb config + +local S = require("syscall") + +function make_handle_prop_args(name, duration, pidbox) + local handler = function(prop_args) + if #prop_args ~= 1 then + print("Usage: snabb quickcheck prop_sameval PCI_ADDR") + os.exit(1) + end + + -- TODO: validate the address + local pci_addr = prop_args[1] + + local pid = S.fork() + if pid == 0 then + local cmdline = {"snabb", "lwaftr", "run", "-D", tostring(duration), + "--conf", "program/lwaftr/tests/data/icmp_on_fail.conf", + "--reconfigurable", "--on-a-stick", pci_addr} + -- FIXME: preserve the environment + S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, {}) + else + pidbox[1] = pid + S.sleep(0.1) + end + end + return handler +end + +function make_cleanup(pidbox) + local cleanup = function() + S.kill(pidbox[1], "TERM") + end + return cleanup +end diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index 2b42abcc49..3b154a87a5 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -2,12 +2,12 @@ module(..., package.seeall) local genyang = require("program.lwaftr.tests.propbased.genyang") -local S = require("syscall") -local run_pid +local common = require("program.lwaftr.tests.propbased.common") +local run_pid = {} local current_cmd function property() - current_cmd = genyang.generate_yang(run_pid) + current_cmd = genyang.generate_get(run_pid[1]) local results = (genyang.run_yang(current_cmd)) if string.match("Could not connect to config leader socket on Snabb instance", results) then @@ -20,28 +20,7 @@ function print_extra_information() print("The command was:", current_cmd) end -function handle_prop_args(prop_args) - if #prop_args ~= 1 then - print("Usage: snabb quickcheck prop_nocrash PCI_ADDR") - os.exit(1) - end - - -- TODO: validate the address - local pci_addr = prop_args[1] - - local pid = S.fork() - if pid == 0 then - local cmdline = {"snabb", "lwaftr", "run", "-D", "20", "--conf", - "program/lwaftr/tests/data/icmp_on_fail.conf", "--reconfigurable", - "--on-a-stick", pci_addr} - -- FIXME: preserve the environment - S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, {}) - else - run_pid = pid - S.sleep(1) - end -end +handle_prop_args = + common.make_handle_prop_args("prop_nocrash", 10, run_pid) -function cleanup() - S.kill(run_pid, "TERM") -end +cleanup = common.make_cleanup(run_pid) diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index 93da4ebaef..864350d3db 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -5,12 +5,12 @@ module(..., package.seeall) -- same values getting returned local genyang = require("program.lwaftr.tests.propbased.genyang") -local S = require("syscall") -local run_pid +local common = require("program.lwaftr.tests.propbased.common") +local run_pid = {} local current_cmd function property() - current_cmd = genyang.generate_yang(run_pid) + current_cmd = genyang.generate_yang(run_pid[1]) local results = (genyang.run_yang(current_cmd)) local results2 = (genyang.run_yang(current_cmd)) if string.match("Could not connect to config leader socket on Snabb instance", @@ -31,28 +31,7 @@ function print_extra_information() print("The command was:", current_cmd) end -function handle_prop_args(prop_args) - if #prop_args ~= 1 then - print("Usage: snabb quickcheck prop_sameval PCI_ADDR") - os.exit(1) - end - - -- TODO: validate the address - local pci_addr = prop_args[1] - - local pid = S.fork() - if pid == 0 then - local cmdline = {"snabb", "lwaftr", "run", "-D", "20", "--conf", - "program/lwaftr/tests/data/icmp_on_fail.conf", "--reconfigurable", - "--on-a-stick", pci_addr} - -- FIXME: preserve the environment - S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, {}) - else - run_pid = pid - S.sleep(0.1) - end -end +handle_prop_args = + common.make_handle_prop_args("prop_sameval", 20, run_pid) -function cleanup() - S.kill(run_pid, "TERM") -end +cleanup = common.make_cleanup(run_pid) From 56d415b3d7734e950121ac6da72aed18d97e62af Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 20:42:07 -0800 Subject: [PATCH 498/631] Add generation of `snabb config set` commands Use in the sameval property test --- .../lwaftr/tests/propbased/genyang.lua | 7 ++++++- .../lwaftr/tests/propbased/prop_sameval.lua | 20 +++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index f7aeb3ffe3..802d593b65 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -6,11 +6,16 @@ module(..., package.seeall) local schema = require("lib.yang.schema") local softwire_schema = schema.load_schema_by_name("snabb-softwire-v1") -function generate_yang(pid) +function generate_get(pid) local query = generate_xpath(softwire_schema) return string.format("./snabb config get %s \"%s\"", pid, query) end +function generate_set(pid, val) + local query = generate_xpath(softwire_schema) + return string.format("./snabb config set %s \"%s\" \"%s\"", pid, query, val) +end + function run_yang(yang_cmd) local f = io.popen(yang_cmd) local result = f:read("*a") diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index 864350d3db..a8084173d4 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -10,12 +10,20 @@ local run_pid = {} local current_cmd function property() - current_cmd = genyang.generate_yang(run_pid[1]) - local results = (genyang.run_yang(current_cmd)) - local results2 = (genyang.run_yang(current_cmd)) + local get = genyang.generate_get(run_pid[1]) + local results = (genyang.run_yang(get)) + + if string.match("Could not connect to config leader socket on Snabb instance", + results) then + print("Launching snabb run failed, or we've crashed it!") + return false + end + + local set = genyang.generate_set(run_pid[1], results) + genyang.run_yang(set) + local results2 = (genyang.run_yang(get)) + if string.match("Could not connect to config leader socket on Snabb instance", - results) or - string.match("Could not connect to config leader socket on Snabb instance", results2) then print("Launching snabb run failed, or we've crashed it!") return false @@ -32,6 +40,6 @@ function print_extra_information() end handle_prop_args = - common.make_handle_prop_args("prop_sameval", 20, run_pid) + common.make_handle_prop_args("prop_sameval", 40, run_pid) cleanup = common.make_cleanup(run_pid) From b5a863c92b311d5707ed81b0a276765f51ec74dd Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 20:52:40 -0800 Subject: [PATCH 499/631] Add a new test for the get-state command --- .../lwaftr/tests/propbased/genyang.lua | 4 +++ .../tests/propbased/prop_nocrash_state.lua | 27 +++++++++++++++++++ .../lwaftr/tests/propbased/selftest.sh | 1 + 3 files changed, 32 insertions(+) create mode 100644 src/program/lwaftr/tests/propbased/prop_nocrash_state.lua diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 802d593b65..ec3b93c700 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -11,6 +11,10 @@ function generate_get(pid) return string.format("./snabb config get %s \"%s\"", pid, query) end +function generate_get_state(pid) + return string.format("./snabb config get-state %s", pid) +end + function generate_set(pid, val) local query = generate_xpath(softwire_schema) return string.format("./snabb config set %s \"%s\" \"%s\"", pid, query, val) diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua b/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua new file mode 100644 index 0000000000..276226e529 --- /dev/null +++ b/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua @@ -0,0 +1,27 @@ +module(..., package.seeall) + +-- test to make sure repeated get-state commands are ok + +local genyang = require("program.lwaftr.tests.propbased.genyang") +local common = require("program.lwaftr.tests.propbased.common") +local run_pid = {} +local current_cmd + +function property() + current_cmd = genyang.generate_get_state(run_pid[1]) + local results = (genyang.run_yang(current_cmd)) + if string.match("Could not connect to config leader socket on Snabb instance", + results) then + print("Launching snabb run failed, or we've crashed it!") + return false + end +end + +function print_extra_information() + print("The command was:", current_cmd) +end + +handle_prop_args = + common.make_handle_prop_args("prop_nocrash_state", 10, run_pid) + +cleanup = common.make_cleanup(run_pid) diff --git a/src/program/lwaftr/tests/propbased/selftest.sh b/src/program/lwaftr/tests/propbased/selftest.sh index d44ea32eba..e313cc6206 100755 --- a/src/program/lwaftr/tests/propbased/selftest.sh +++ b/src/program/lwaftr/tests/propbased/selftest.sh @@ -7,4 +7,5 @@ SKIPPED_CODE=43 if [ -z $SNABB_PCI0 ]; then exit $SKIPPED_CODE; fi ./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash $SNABB_PCI0 +./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash_state $SNABB_PCI0 ./snabb quickcheck program.lwaftr.tests.propbased.prop_sameval $SNABB_PCI0 From 1c4a26fb68e3e3bb6ed7a58b9d811dc804582d38 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 9 Dec 2016 20:57:45 -0800 Subject: [PATCH 500/631] Make get-state lookups use generated queries too --- src/program/lwaftr/tests/propbased/genyang.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index ec3b93c700..a11228d8c1 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -12,7 +12,8 @@ function generate_get(pid) end function generate_get_state(pid) - return string.format("./snabb config get-state %s", pid) + local query = generate_xpath_state(softwire_schema, true) + return string.format("./snabb config get-state %s \"%s\"", pid, query) end function generate_set(pid, val) @@ -54,6 +55,10 @@ local function choose_pos() end end +function generate_xpath_state() + return generate_xpath(softwire_schema.body["softwire-state"]) +end + -- from a config schema, generate an xpath query string -- this code is patterned off of the visitor used in lib.yang.data function generate_xpath(schema) From 3b766386bec3da50167e0bf99d6aa8f8bbf228de Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Mon, 12 Dec 2016 20:56:55 +0000 Subject: [PATCH 501/631] Fix the get-state test to have non-empty query results The empty environment for running the lwaftr was causing the state query to return an empty result. The PATH variable needs to be preserved at least. Also adjust the xpath generation to accommodate the get-state command. --- src/program/lwaftr/tests/propbased/common.lua | 7 +++++-- src/program/lwaftr/tests/propbased/genyang.lua | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/common.lua b/src/program/lwaftr/tests/propbased/common.lua index 733e3e1ff2..059bcd45fd 100644 --- a/src/program/lwaftr/tests/propbased/common.lua +++ b/src/program/lwaftr/tests/propbased/common.lua @@ -19,8 +19,11 @@ function make_handle_prop_args(name, duration, pidbox) local cmdline = {"snabb", "lwaftr", "run", "-D", tostring(duration), "--conf", "program/lwaftr/tests/data/icmp_on_fail.conf", "--reconfigurable", "--on-a-stick", pci_addr} - -- FIXME: preserve the environment - S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, {}) + -- preserve PATH variable because the get-state test relies on + -- this variable being set to print useful results + local pth = os.getenv("PATH") + local env = { ("PATH=%s"):format(pth) } + S.execve(("/proc/%d/exe"):format(S.getpid()), cmdline, env) else pidbox[1] = pid S.sleep(0.1) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index a11228d8c1..36193c8434 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -56,12 +56,13 @@ local function choose_pos() end function generate_xpath_state() - return generate_xpath(softwire_schema.body["softwire-state"]) + local path = generate_xpath(softwire_schema.body["softwire-state"], true) + return "/softwire-state" .. path end -- from a config schema, generate an xpath query string -- this code is patterned off of the visitor used in lib.yang.data -function generate_xpath(schema) +function generate_xpath(schema, for_state) local path = "" local handlers = {} @@ -72,8 +73,9 @@ function generate_xpath(schema) local function visit_body(node) local ids = {} for id, node in pairs(node.body) do - -- only choose nodes that are used in configs - if node.config ~= false then + -- only choose nodes that are used in configs unless + -- for_state is passed + if for_state or node.config ~= false then table.insert(ids, id) end end From 7e018bc82b2a841c87097fedc1a7b4c8002f1fef Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Mon, 12 Dec 2016 13:09:46 -0800 Subject: [PATCH 502/631] Bump get-state test duration --- src/program/lwaftr/tests/propbased/prop_nocrash_state.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua b/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua index 276226e529..c897551abd 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua @@ -22,6 +22,6 @@ function print_extra_information() end handle_prop_args = - common.make_handle_prop_args("prop_nocrash_state", 10, run_pid) + common.make_handle_prop_args("prop_nocrash_state", 20, run_pid) cleanup = common.make_cleanup(run_pid) From ae6c8b9382e95a639a189342b53fe664bbb16ff6 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Mon, 12 Dec 2016 13:37:36 -0800 Subject: [PATCH 503/631] Add optional query parameter to generate functions Also change the API to export generation functions that hardcode the right schema for these tests. Use the optional parameter to fix the sameval property (it wasn't actually setting the same property correctly) --- .../lwaftr/tests/propbased/genyang.lua | 31 ++++++++++++------- .../lwaftr/tests/propbased/prop_sameval.lua | 5 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 36193c8434..1a9bbf1d48 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -6,18 +6,21 @@ module(..., package.seeall) local schema = require("lib.yang.schema") local softwire_schema = schema.load_schema_by_name("snabb-softwire-v1") -function generate_get(pid) - local query = generate_xpath(softwire_schema) +function generate_get(pid, query) + if not query then + query = generate_config_xpath() + end return string.format("./snabb config get %s \"%s\"", pid, query) end -function generate_get_state(pid) - local query = generate_xpath_state(softwire_schema, true) +function generate_get_state(pid, query) + if not query then + query = generate_config_xpath_state() + end return string.format("./snabb config get-state %s \"%s\"", pid, query) end -function generate_set(pid, val) - local query = generate_xpath(softwire_schema) +function generate_set(pid, query, val) return string.format("./snabb config set %s \"%s\" \"%s\"", pid, query, val) end @@ -55,14 +58,9 @@ local function choose_pos() end end -function generate_xpath_state() - local path = generate_xpath(softwire_schema.body["softwire-state"], true) - return "/softwire-state" .. path -end - -- from a config schema, generate an xpath query string -- this code is patterned off of the visitor used in lib.yang.data -function generate_xpath(schema, for_state) +local function generate_xpath(schema, for_state) local path = "" local handlers = {} @@ -116,6 +114,15 @@ function generate_xpath(schema, for_state) return path end +function generate_config_xpath() + return generate_xpath(softwire_schema, false) +end + +function generate_config_xpath_state() + local path = generate_xpath(softwire_schema.body["softwire-state"], true) + return "/softwire-state" .. path +end + function selftest() local data = require("lib.yang.data") local path = require("lib.yang.path") diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index a8084173d4..2cfdc81b94 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -10,7 +10,8 @@ local run_pid = {} local current_cmd function property() - local get = genyang.generate_get(run_pid[1]) + local xpath = genyang.generate_config_xpath() + local get = genyang.generate_get(run_pid[1], xpath) local results = (genyang.run_yang(get)) if string.match("Could not connect to config leader socket on Snabb instance", @@ -19,7 +20,7 @@ function property() return false end - local set = genyang.generate_set(run_pid[1], results) + local set = genyang.generate_set(run_pid[1], xpath, results) genyang.run_yang(set) local results2 = (genyang.run_yang(get)) From 62f97a5a8c8581f333efaa3cb878bcbebe04547e Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Tue, 13 Dec 2016 04:05:37 +0000 Subject: [PATCH 504/631] Improve sameval test to be resilient to missing fields Sometimes the generated queries are for fields that are missing in the configuration. Ignore those cases for doing a subsequent "set". Also abstracted the crash handling code. --- .../lwaftr/tests/propbased/prop_sameval.lua | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index 2cfdc81b94..e32fcd1016 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -9,24 +9,43 @@ local common = require("program.lwaftr.tests.propbased.common") local run_pid = {} local current_cmd +local function check_crashed(results) + if results:match("Could not connect to config leader socket on Snabb instance") then + print("Launching snabb run failed, or we've crashed it!") + return true + end + return false +end + function property() local xpath = genyang.generate_config_xpath() local get = genyang.generate_get(run_pid[1], xpath) + current_cmd = get + local results = (genyang.run_yang(get)) - if string.match("Could not connect to config leader socket on Snabb instance", - results) then - print("Launching snabb run failed, or we've crashed it!") + if check_crashed(results) then return false end + -- queried data doesn't exist most likely (or some other non-fatal error) + if results:match("short read") then + -- just continue because it's not worth trying to set this property + return + end + local set = genyang.generate_set(run_pid[1], xpath, results) - genyang.run_yang(set) + results_set = genyang.run_yang(set) + current_cmd = set + + if check_crashed(results_set) then + return false + end + local results2 = (genyang.run_yang(get)) + current_cmd = get - if string.match("Could not connect to config leader socket on Snabb instance", - results2) then - print("Launching snabb run failed, or we've crashed it!") + if check_crashed(results2) then return false end From d3502329328a53f76300f8c71bfce14487e63631 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Tue, 13 Dec 2016 04:17:00 +0000 Subject: [PATCH 505/631] Abstract out the crash checking into common.lua --- src/program/lwaftr/tests/propbased/common.lua | 9 +++++++++ .../lwaftr/tests/propbased/prop_nocrash.lua | 4 +--- .../lwaftr/tests/propbased/prop_nocrash_state.lua | 4 +--- .../lwaftr/tests/propbased/prop_sameval.lua | 14 +++----------- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/common.lua b/src/program/lwaftr/tests/propbased/common.lua index 059bcd45fd..a5f7696a5b 100644 --- a/src/program/lwaftr/tests/propbased/common.lua +++ b/src/program/lwaftr/tests/propbased/common.lua @@ -38,3 +38,12 @@ function make_cleanup(pidbox) end return cleanup end + +-- return true if the result from the query indicates a crash/disconnect +function check_crashed(results) + if results:match("Could not connect to config leader socket on Snabb instance") then + print("Launching snabb run failed, or we've crashed it!") + return true + end + return false +end diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index 3b154a87a5..505d0645d1 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -9,9 +9,7 @@ local current_cmd function property() current_cmd = genyang.generate_get(run_pid[1]) local results = (genyang.run_yang(current_cmd)) - if string.match("Could not connect to config leader socket on Snabb instance", - results) then - print("Launching snabb run failed, or we've crashed it!") + if common.check_crashed(results) then return false end end diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua b/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua index c897551abd..4a13fac7dd 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash_state.lua @@ -10,9 +10,7 @@ local current_cmd function property() current_cmd = genyang.generate_get_state(run_pid[1]) local results = (genyang.run_yang(current_cmd)) - if string.match("Could not connect to config leader socket on Snabb instance", - results) then - print("Launching snabb run failed, or we've crashed it!") + if common.check_crashed(results) then return false end end diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index e32fcd1016..fb267ac42a 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -9,14 +9,6 @@ local common = require("program.lwaftr.tests.propbased.common") local run_pid = {} local current_cmd -local function check_crashed(results) - if results:match("Could not connect to config leader socket on Snabb instance") then - print("Launching snabb run failed, or we've crashed it!") - return true - end - return false -end - function property() local xpath = genyang.generate_config_xpath() local get = genyang.generate_get(run_pid[1], xpath) @@ -24,7 +16,7 @@ function property() local results = (genyang.run_yang(get)) - if check_crashed(results) then + if common.check_crashed(results) then return false end @@ -38,14 +30,14 @@ function property() results_set = genyang.run_yang(set) current_cmd = set - if check_crashed(results_set) then + if common.check_crashed(results_set) then return false end local results2 = (genyang.run_yang(get)) current_cmd = get - if check_crashed(results2) then + if common.check_crashed(results2) then return false end From 5cb065e23b9cae83a53951608c7fa3c04db053df Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 14 Dec 2016 01:02:12 +0000 Subject: [PATCH 506/631] Generate selectors for list indexing in tests --- .../lwaftr/tests/propbased/genyang.lua | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 1a9bbf1d48..729aad2827 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -58,6 +58,51 @@ local function choose_pos() end end +local function value_from_type(a_type) + local prim = a_type.primitive_type + + if prim == "int8" then + return math.random(-128, 127) + elseif prim == "int16" then + return math.random(-32768, 32767) + elseif prim == "int32" then + return math.random(-2147483648, 2147483647) + elseif prim == "int64" then + return math.random(-9223372036854775808, 9223372036854775807) + elseif prim == "uint8" then + return math.random(0, 255) + elseif prim == "uint16" then + return math.random(0, 65535) + elseif prim == "uint32" then + return math.random(0, 4294967295) + elseif prim == "uint64" then + return math.random(0, 18446744073709551615) + --elseif prim == "decimal64" then + -- local int64 = value_from_type("int64") + -- local exp = math.random(1, 18) + -- return int64 * (10 ^ -exp) + elseif prim == "boolean" then + return choose({ true, false }) + elseif prim == "ipv4-address" then + return math.random(0, 255) .. "." .. math.random(0, 255) .. "." .. + math.random(0, 255) .. "." .. math.random(0, 255) + end + + -- TODO: generate these: + -- string + -- binary + -- bits + -- empty + -- enumeration + -- identityref + -- instance-identifier + -- leafref + -- union + + -- unknown type + return nil +end + -- from a config schema, generate an xpath query string -- this code is patterned off of the visitor used in lib.yang.data local function generate_xpath(schema, for_state) @@ -97,10 +142,22 @@ local function generate_xpath(schema, for_state) path = path .. "/" .. node.id .. selector end function handlers.list(node) - -- TODO: this should generate selector-based lookups by using - -- the key type as a generation source, but for now do - -- the simple thing + local key_types = {} + local r = math.random() + path = path .. "/" .. node.id + + -- occasionally drop the selectors + if r < 0.9 then + for key in (node.key):split(" +") do + key_types[key] = node.body[key].type + end + + for key, type in pairs(key_types) do + local val = assert(value_from_type(type), type) + path = path .. string.format("[%s=%s]", key, val) + end + end end function handlers.leaf(node) path = path .. "/" .. node.id From 65af1cd8e8fb4b2a62439519c2cebb9cd02629ff Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 14 Dec 2016 22:56:27 +0000 Subject: [PATCH 507/631] Test the ietf-softwire config protocol too --- .../lwaftr/tests/propbased/genyang.lua | 46 ++++++++++++------- .../lwaftr/tests/propbased/prop_sameval.lua | 6 +-- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 729aad2827..15220509cf 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -4,24 +4,29 @@ module(..., package.seeall) --local snabb_cmd = ("/proc/%d/exe"):format(S.getpid()) local schema = require("lib.yang.schema") -local softwire_schema = schema.load_schema_by_name("snabb-softwire-v1") -function generate_get(pid, query) +local capabilities = {['ietf-softwire']={feature={'binding', 'br'}}} +require('lib.yang.schema').set_default_capabilities(capabilities) + +local schemas = { "ietf-softwire", "snabb-softwire-v1" } + +function generate_get(pid, schema, query) if not query then - query = generate_config_xpath() + query, schema = generate_config_xpath(schema) end - return string.format("./snabb config get %s \"%s\"", pid, query) + return string.format("./snabb config get -s %s %s \"%s\"", schema, pid, query) end -function generate_get_state(pid, query) +function generate_get_state(pid, schema, query) if not query then - query = generate_config_xpath_state() + query, schema = generate_config_xpath_state(schema) end - return string.format("./snabb config get-state %s \"%s\"", pid, query) + return string.format("./snabb config get-state -s %s %s \"%s\"", schema, pid, query) end -function generate_set(pid, query, val) - return string.format("./snabb config set %s \"%s\" \"%s\"", pid, query, val) +function generate_set(pid, schema, query, val) + return string.format("./snabb config set -s %s %s \"%s\" \"%s\"", + schema, pid, query, val) end function run_yang(yang_cmd) @@ -171,19 +176,28 @@ local function generate_xpath(schema, for_state) return path end -function generate_config_xpath() - return generate_xpath(softwire_schema, false) +function generate_config_xpath(schema_name) + if not schema_name then + schema_name = choose(schemas) + end + local schema = schema.load_schema_by_name(schema_name) + return generate_xpath(schema, false), schema_name end -function generate_config_xpath_state() - local path = generate_xpath(softwire_schema.body["softwire-state"], true) - return "/softwire-state" .. path +function generate_config_xpath_state(schema_name) + if not schema_name then + schema_name = choose(schemas) + end + local schema = schema.load_schema_by_name(schema_name) + local path = generate_xpath(schema.body["softwire-state"], true) + return "/softwire-state" .. path, schema_name end function selftest() local data = require("lib.yang.data") local path = require("lib.yang.path") - local grammar = data.data_grammar_from_schema(softwire_schema) + local schema = schema.load_schema_by_name("snabb-softwire-v1") + local grammar = data.data_grammar_from_schema(schema) - path.convert_path(grammar, generate_xpath(softwire_schema)) + path.convert_path(grammar, generate_xpath(schema)) end diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index fb267ac42a..e05f442d16 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -10,8 +10,8 @@ local run_pid = {} local current_cmd function property() - local xpath = genyang.generate_config_xpath() - local get = genyang.generate_get(run_pid[1], xpath) + local xpath, schema_name = genyang.generate_config_xpath() + local get = genyang.generate_get(run_pid[1], schema_name, xpath) current_cmd = get local results = (genyang.run_yang(get)) @@ -26,7 +26,7 @@ function property() return end - local set = genyang.generate_set(run_pid[1], xpath, results) + local set = genyang.generate_set(run_pid[1], schema_name, xpath, results) results_set = genyang.run_yang(set) current_cmd = set From 815c258684749cba413e55b00e3c587404d54a17 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 16 Dec 2016 00:29:57 +0000 Subject: [PATCH 508/631] Traverse 'list' nodes for random generation better Also generate values for ipv6 addresses/prefixes and unions to better explore the schema --- .../lwaftr/tests/propbased/genyang.lua | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 15220509cf..19ecfd80b2 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -63,6 +63,14 @@ local function choose_pos() end end +local function random_hexes() + local str = "" + for i=1, 4 do + str = str .. string.format("%x", math.random(0, 15)) + end + return str +end + local function value_from_type(a_type) local prim = a_type.primitive_type @@ -91,6 +99,17 @@ local function value_from_type(a_type) elseif prim == "ipv4-address" then return math.random(0, 255) .. "." .. math.random(0, 255) .. "." .. math.random(0, 255) .. "." .. math.random(0, 255) + elseif prim == "ipv6-address" then + local addr = random_hexes() + for i=1, 7 do + addr = addr .. ":" .. random_hexes() + end + return addr + elseif prim == "ipv6-prefix" then + local addr = value_from_type({ primitive_type = "ipv6-address" }) + return addr .. "/" .. math.random(0, 128) + elseif prim == "union" then + return value_from_type(choose(a_type.union)) end -- TODO: generate these: @@ -102,7 +121,6 @@ local function value_from_type(a_type) -- identityref -- instance-identifier -- leafref - -- union -- unknown type return nil @@ -159,10 +177,14 @@ local function generate_xpath(schema, for_state) end for key, type in pairs(key_types) do - local val = assert(value_from_type(type), type) + local val = assert(value_from_type(type), type.primitive_type) path = path .. string.format("[%s=%s]", key, val) end end + + if math.random() < 0.9 then + visit_body(node) + end end function handlers.leaf(node) path = path .. "/" .. node.id From e42ad26688065210651df5f79758a5c89f9b751e Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Sat, 17 Dec 2016 02:00:10 +0000 Subject: [PATCH 509/631] Adjust how integer values are generated to prefer boundaries --- .../lwaftr/tests/propbased/genyang.lua | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 19ecfd80b2..8cabe8e053 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -71,25 +71,36 @@ local function random_hexes() return str end +-- return a random number, preferring boundary values +local function choose_range(lo, hi) + local r = math.random() + if r < 0.5 then + local mid = math.ceil((hi + lo) / 2) + return choose({ lo, lo + 1, mid, mid + 1, hi - 1, hi }) + else + return math.random(lo, hi) + end +end + local function value_from_type(a_type) local prim = a_type.primitive_type if prim == "int8" then - return math.random(-128, 127) + return choose_range(-128, 127) elseif prim == "int16" then - return math.random(-32768, 32767) + return choose_range(-32768, 32767) elseif prim == "int32" then - return math.random(-2147483648, 2147483647) + return choose_range(-2147483648, 2147483647) elseif prim == "int64" then - return math.random(-9223372036854775808, 9223372036854775807) + return choose_range(-9223372036854775809, 9223372036854775807) elseif prim == "uint8" then - return math.random(0, 255) + return choose_range(0, 255) elseif prim == "uint16" then - return math.random(0, 65535) + return choose_range(0, 65535) elseif prim == "uint32" then - return math.random(0, 4294967295) + return choose_range(0, 4294967295) elseif prim == "uint64" then - return math.random(0, 18446744073709551615) + return choose_range(0, 18446744073709551615) --elseif prim == "decimal64" then -- local int64 = value_from_type("int64") -- local exp = math.random(1, 18) From 905a151a1edf92e6688d8a7888b1a452ce5187d8 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Tue, 20 Dec 2016 03:11:42 +0000 Subject: [PATCH 510/631] Prefer lower position arguments --- src/program/lwaftr/tests/propbased/genyang.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 8cabe8e053..c188737e5e 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -55,11 +55,11 @@ local function choose_pos() end end - -- evenly weight first five indices + -- evenly weight first two if r < 0.5 then - return choose({1, 2, 3, 4, 5}) + return choose({1, 2}) else - return flip(6) + return flip(3) end end From d7460646a670026e11ccbbcaa3a72425003897c5 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 21 Dec 2016 02:53:35 +0000 Subject: [PATCH 511/631] Start to add functions for generating set/add values Generates values from the schema fo a given path for use with the config set and add functions. --- .../lwaftr/tests/propbased/genyang.lua | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index c188737e5e..4f9e1b46b5 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -10,6 +10,18 @@ require('lib.yang.schema').set_default_capabilities(capabilities) local schemas = { "ietf-softwire", "snabb-softwire-v1" } +function generate_get_or_set(pid, schema) + local r = math.random() + if r > 0.5 then + local query, schema = generate_config_xpath(schema) + return string.format("./snabb config get -s %s %s \"%s\"", schema, pid, query) + else + local query, val, schema = generate_config_xpath_and_val(schema) + return string.format("./snabb config set -s %s %s \"%s\" \"%s\"", + schema, pid, query, val) + end +end + function generate_get(pid, schema, query) if not query then query, schema = generate_config_xpath(schema) @@ -139,9 +151,10 @@ end -- from a config schema, generate an xpath query string -- this code is patterned off of the visitor used in lib.yang.data -local function generate_xpath(schema, for_state) +local function generate_xpath_and_last_node(schema, for_state) local path = "" local handlers = {} + local last_node local function visit(node) local handler = handlers[node.kind] @@ -160,6 +173,8 @@ local function generate_xpath(schema, for_state) local id = choose(ids) if id then visit(node.body[id]) + else + last_node = node end end function handlers.container(node) @@ -169,11 +184,14 @@ local function generate_xpath(schema, for_state) -- fetching all sub-items too if math.random() < 0.9 then visit_body(node) + else + last_node = node end end handlers['leaf-list'] = function(node) local selector = string.format("[position()=%d]", choose_pos()) path = path .. "/" .. node.id .. selector + last_node = node end function handlers.list(node) local key_types = {} @@ -195,10 +213,14 @@ local function generate_xpath(schema, for_state) if math.random() < 0.9 then visit_body(node) + else + last_node = node end end function handlers.leaf(node) path = path .. "/" .. node.id + val = value_from_type(node.type) + last_node = node end -- just produce "/" on rare occasions @@ -206,9 +228,28 @@ local function generate_xpath(schema, for_state) visit_body(schema) end + return path, last_node +end + +local function generate_xpath(schema, for_state) + local path = generate_xpath_and_last_node(schema, for_state) return path end +local function generate_xpath_and_val(schema) + local val, path, last + + while not val do + path, last = generate_xpath_and_last_node(schema) + + if last and last.kind == "leaf" then + val = value_from_type(last.type) + end + end + + return path, val +end + function generate_config_xpath(schema_name) if not schema_name then schema_name = choose(schemas) @@ -217,6 +258,15 @@ function generate_config_xpath(schema_name) return generate_xpath(schema, false), schema_name end +function generate_config_xpath_and_val(schema_name) + if not schema_name then + schema_name = choose(schemas) + end + local schema = schema.load_schema_by_name(schema_name) + local path, val = generate_xpath_and_val(schema) + return path, val, schema_name +end + function generate_config_xpath_state(schema_name) if not schema_name then schema_name = choose(schemas) From d938919a00d1ad196491c6194c4b85a20c6155e8 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 4 Jan 2017 20:39:33 +0000 Subject: [PATCH 512/631] Use new set command generation for nocrash test And bump test duration --- src/program/lwaftr/tests/propbased/prop_nocrash.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index 505d0645d1..57c3da673e 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -7,7 +7,7 @@ local run_pid = {} local current_cmd function property() - current_cmd = genyang.generate_get(run_pid[1]) + current_cmd = genyang.generate_get_or_set(run_pid[1]) local results = (genyang.run_yang(current_cmd)) if common.check_crashed(results) then return false @@ -19,6 +19,6 @@ function print_extra_information() end handle_prop_args = - common.make_handle_prop_args("prop_nocrash", 10, run_pid) + common.make_handle_prop_args("prop_nocrash", 20, run_pid) cleanup = common.make_cleanup(run_pid) From 341cdeb859ffa6912b767dd1b1761f7f48ac55b4 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Thu, 5 Jan 2017 06:54:05 +0000 Subject: [PATCH 513/631] Initial attempt at generating compound values --- .../lwaftr/tests/propbased/genyang.lua | 60 ++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 4f9e1b46b5..7e2331d8e1 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -231,6 +231,62 @@ local function generate_xpath_and_last_node(schema, for_state) return path, last_node end +local function generate_value_for_node(node) + local handlers = {} + + local function visit(node) + local handler = handlers[node.kind] + if handler then return handler(node) end + end + local function visit_body(node) + local ids = {} + for id, node in pairs(node.body) do + -- only choose nodes that are used in configs + if node.config ~= false then + table.insert(ids, id) + end + end + + local val = "" + for _, id in ipairs(ids) do + local subnode = node.body[id] + local r = math.random() + if subnode.mandatory or r > 0.5 then + + if subnode.kind == "leaf-list" then + local count = choose_pos() -- how many to generate + for i=0, count do + local subval = visit(subnode) + val = val .. string.format("%s %s; ", id, subval) + end + elseif subnode.kind == "container" or subnode.kind == "list" then + local subval = visit(subnode) + val = val .. string.format("%s {%s} ", id, subval) + else + local subval = visit(subnode) + val = val .. string.format("%s %s; ", id, subval) + end + end + end + + return val + end + function handlers.container(node) + return visit_body(node) + end + handlers['leaf-list'] = function(node) + return value_from_type(node.type) + end + function handlers.list(node) + return visit_body(node) + end + function handlers.leaf(node) + return value_from_type(node.type) + end + + return visit(node) +end + local function generate_xpath(schema, for_state) local path = generate_xpath_and_last_node(schema, for_state) return path @@ -242,8 +298,8 @@ local function generate_xpath_and_val(schema) while not val do path, last = generate_xpath_and_last_node(schema) - if last and last.kind == "leaf" then - val = value_from_type(last.type) + if last then + val = generate_value_for_node(last) end end From ece5893db0f6b9a53d93cd43a8c802a2bc410ffb Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Thu, 5 Jan 2017 19:54:28 +0000 Subject: [PATCH 514/631] Generate more leaf data types --- .../lwaftr/tests/propbased/genyang.lua | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 7e2331d8e1..1a57f32dec 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -75,10 +75,14 @@ local function choose_pos() end end +local function random_hex() + return string.format("%x", math.random(0, 15)) +end + local function random_hexes() local str = "" for i=1, 4 do - str = str .. string.format("%x", math.random(0, 15)) + str = str .. random_hex() end return str end @@ -131,8 +135,22 @@ local function value_from_type(a_type) elseif prim == "ipv6-prefix" then local addr = value_from_type({ primitive_type = "ipv6-address" }) return addr .. "/" .. math.random(0, 128) + elseif prim == "mac-address" then + local addr = random_hex() .. random_hex() + for i=1,5 do + addr = addr .. ":" .. random_hex() .. random_hex() + end + return addr elseif prim == "union" then return value_from_type(choose(a_type.union)) + elseif prim == "string" then + local len = choose_pos() + -- just ascii for now + local str = "" + for i=0, len do + str = str .. string.char(math.random(97, 122)) + end + return str end -- TODO: generate these: From 9b3baf6fd031b8115868a80b11cb94b2b818ee1b Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 6 Jan 2017 01:35:35 +0000 Subject: [PATCH 515/631] Add some comments to genyang.lua --- src/program/lwaftr/tests/propbased/genyang.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 1a57f32dec..9f3acc0636 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -249,6 +249,9 @@ local function generate_xpath_and_last_node(schema, for_state) return path, last_node end +-- similar to generating a query path like the function above, but +-- generates a compound value for `snabb config set` at some schema +-- node local function generate_value_for_node(node) local handlers = {} @@ -296,6 +299,8 @@ local function generate_value_for_node(node) return value_from_type(node.type) end function handlers.list(node) + -- FIXME: this will sometimes include a value for the list keys + -- which isn't valid when the query path sets the keys return visit_body(node) end function handlers.leaf(node) From 000ae61d39d2ee5cf786a5e4a20b818dd299fe3c Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 6 Jan 2017 06:10:32 +0000 Subject: [PATCH 516/631] Add comments, minor refactoring --- .../lwaftr/tests/propbased/genyang.lua | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 9f3acc0636..4a647ba89e 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -1,7 +1,7 @@ module(..., package.seeall) ---local S = require("syscall") ---local snabb_cmd = ("/proc/%d/exe"):format(S.getpid()) +-- This module provides functions for generating snabb config +-- commands with random path queries and values local schema = require("lib.yang.schema") @@ -10,6 +10,7 @@ require('lib.yang.schema').set_default_capabilities(capabilities) local schemas = { "ietf-softwire", "snabb-softwire-v1" } +-- Generate a get/set command string given a pid string and optional schema function generate_get_or_set(pid, schema) local r = math.random() if r > 0.5 then @@ -22,6 +23,7 @@ function generate_get_or_set(pid, schema) end end +-- Generate a get command string given a pid string and optional schema/query function generate_get(pid, schema, query) if not query then query, schema = generate_config_xpath(schema) @@ -29,6 +31,7 @@ function generate_get(pid, schema, query) return string.format("./snabb config get -s %s %s \"%s\"", schema, pid, query) end +-- Like generate_get but for state queries function generate_get_state(pid, schema, query) if not query then query, schema = generate_config_xpath_state(schema) @@ -36,6 +39,7 @@ function generate_get_state(pid, schema, query) return string.format("./snabb config get-state -s %s %s \"%s\"", schema, pid, query) end +-- Used primarily for repeating a set with a value seen before from a get function generate_set(pid, schema, query, val) return string.format("./snabb config set -s %s %s \"%s\" \"%s\"", schema, pid, query, val) @@ -54,8 +58,9 @@ local function choose(choices) return choices[idx] end --- choose from unbounded array indices, decreasing likelihood -local function choose_pos() +-- choose a natural number (e.g., index or length of array) by +-- repeating a cointoss +local function choose_nat() local r = math.random() local function flip(next) @@ -144,7 +149,7 @@ local function value_from_type(a_type) elseif prim == "union" then return value_from_type(choose(a_type.union)) elseif prim == "string" then - local len = choose_pos() + local len = choose_nat() -- just ascii for now local str = "" for i=0, len do @@ -154,7 +159,6 @@ local function value_from_type(a_type) end -- TODO: generate these: - -- string -- binary -- bits -- empty @@ -207,7 +211,7 @@ local function generate_xpath_and_last_node(schema, for_state) end end handlers['leaf-list'] = function(node) - local selector = string.format("[position()=%d]", choose_pos()) + local selector = string.format("[position()=%d]", choose_nat()) path = path .. "/" .. node.id .. selector last_node = node end @@ -275,7 +279,7 @@ local function generate_value_for_node(node) if subnode.mandatory or r > 0.5 then if subnode.kind == "leaf-list" then - local count = choose_pos() -- how many to generate + local count = choose_nat() for i=0, count do local subval = visit(subnode) val = val .. string.format("%s %s; ", id, subval) From 4f73d09f0a804c86765f918adc1f38a6e890b934 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 11 Jan 2017 20:58:07 +0000 Subject: [PATCH 517/631] Adjust weight for boundary values in number ranges --- src/program/lwaftr/tests/propbased/genyang.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 4a647ba89e..42842555b2 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -95,9 +95,9 @@ end -- return a random number, preferring boundary values local function choose_range(lo, hi) local r = math.random() - if r < 0.5 then + if r < 0.1 then local mid = math.ceil((hi + lo) / 2) - return choose({ lo, lo + 1, mid, mid + 1, hi - 1, hi }) + return choose({ 0, lo, lo + 1, mid, mid + 1, hi - 1, hi }) else return math.random(lo, hi) end From 9f9b74b4e4c34394465bc88dcafeee3e189e2d52 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Thu, 12 Jan 2017 22:53:37 +0000 Subject: [PATCH 518/631] Support decimal64 and binary types for test gen --- .../lwaftr/tests/propbased/genyang.lua | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 42842555b2..744b0f4be1 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -122,10 +122,11 @@ local function value_from_type(a_type) return choose_range(0, 4294967295) elseif prim == "uint64" then return choose_range(0, 18446744073709551615) - --elseif prim == "decimal64" then - -- local int64 = value_from_type("int64") - -- local exp = math.random(1, 18) - -- return int64 * (10 ^ -exp) + elseif prim == "decimal64" then + local int64 = value_from_type({ primitive_type="int64" }) + local exp = math.random(1, 18) + -- see RFC 6020 sec 9.3.1 for lexical representation + return string.format("%f", int64 * (10 ^ -exp)) elseif prim == "boolean" then return choose({ true, false }) elseif prim == "ipv4-address" then @@ -156,10 +157,36 @@ local function value_from_type(a_type) str = str .. string.char(math.random(97, 122)) end return str + elseif prim == "binary" then + -- TODO: if restricted with length statement this should pick based + -- on the octet length instead and introduce padding chars + -- if necessary + local encoded = "" + local encoded_len = choose_nat() * 4 + + for i=1, encoded_len do + local r = math.random(0, 63) + local byte + + if r <= 25 then + byte = string.byte("A") + r + elseif r > 25 and r <= 51 then + byte = string.byte("a") + r-26 + elseif r > 51 and r <= 61 then + byte = string.byte("0") + r-52 + elseif r == 63 then + byte = string.byte("+") + else + byte = string.byte("/") + end + + encoded = encoded .. string.char(byte) + end + + return encoded end -- TODO: generate these: - -- binary -- bits -- empty -- enumeration @@ -366,4 +393,17 @@ function selftest() local grammar = data.data_grammar_from_schema(schema) path.convert_path(grammar, generate_xpath(schema)) + + -- ensure decimal64 values match the right regexp + for i=1, 100 do + local val = value_from_type({ primitive_type="decimal64" }) + assert(string.match(val, "^-?%d+[.]%d+$"), string.format("test value: %s", val)) + end + + -- ensure generated base64 values are decodeable + for i=1, 100 do + local val = value_from_type({ primitive_type="binary" }) + local cmd = string.format("echo \"%s\" | base64 -d > /dev/null", val) + assert(os.execute(cmd) == 0, string.format("test value: %s", val)) + end end From b5b7aa10fce2f439043d01ffa3f0e5262b12b711 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Thu, 12 Jan 2017 22:54:00 +0000 Subject: [PATCH 519/631] Run more sameval get/set tests on occasion --- .../lwaftr/tests/propbased/prop_sameval.lua | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index e05f442d16..986827dcd8 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -12,38 +12,52 @@ local current_cmd function property() local xpath, schema_name = genyang.generate_config_xpath() local get = genyang.generate_get(run_pid[1], schema_name, xpath) + local iters = 1 + local results, results2 current_cmd = get - local results = (genyang.run_yang(get)) - - if common.check_crashed(results) then - return false + -- occasionally do a bunch of gets/sets at once + if math.random() < 0.01 then + iters = math.random(100, 1000) end - -- queried data doesn't exist most likely (or some other non-fatal error) - if results:match("short read") then - -- just continue because it's not worth trying to set this property - return + for i=1, iters do + results = (genyang.run_yang(get)) + + if common.check_crashed(results) then + return false + end + + -- queried data doesn't exist most likely (or some other non-fatal error) + if results:match("short read") then + -- just continue because it's not worth trying to set this property + return + end end local set = genyang.generate_set(run_pid[1], schema_name, xpath, results) - results_set = genyang.run_yang(set) current_cmd = set - if common.check_crashed(results_set) then - return false + for i=1, iters do + results_set = genyang.run_yang(set) + + if common.check_crashed(results_set) then + return false + end end - local results2 = (genyang.run_yang(get)) current_cmd = get + for i=1, iters do + results2 = (genyang.run_yang(get)) - if common.check_crashed(results2) then - return false - end + if common.check_crashed(results2) then + return false + end - if results ~= results2 then - print("Running the same config command twice produced different outputs") - return false + if results ~= results2 then + print("Running the same config command twice produced different outputs") + return false + end end end @@ -52,6 +66,6 @@ function print_extra_information() end handle_prop_args = - common.make_handle_prop_args("prop_sameval", 40, run_pid) + common.make_handle_prop_args("prop_sameval", 60, run_pid) cleanup = common.make_cleanup(run_pid) From 854d7e64c46c66687708ec1396afdd29a27e3f72 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 13 Jan 2017 00:44:29 +0000 Subject: [PATCH 520/631] Implement enum generation, change error behavior --- src/program/lwaftr/tests/propbased/genyang.lua | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 744b0f4be1..bd28029ec6 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -149,6 +149,7 @@ local function value_from_type(a_type) return addr elseif prim == "union" then return value_from_type(choose(a_type.union)) + -- TODO: follow pattern statement elseif prim == "string" then local len = choose_nat() -- just ascii for now @@ -184,18 +185,21 @@ local function value_from_type(a_type) end return encoded + elseif prim == "empty" then + return "" + elseif prim == "enumeration" then + local enum = choose(a_type.enums) + return enum.value end - -- TODO: generate these: + -- TODO: these appear unused in the current YANG schemas so + -- they're left out for now -- bits - -- empty - -- enumeration -- identityref -- instance-identifier -- leafref - -- unknown type - return nil + error("NYI or unknown type") end -- from a config schema, generate an xpath query string From fe834417060704c987f0fd0fb9cf3c8b15f0a020 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 13 Jan 2017 22:28:37 +0000 Subject: [PATCH 521/631] Use range statements in YANG types for value gen --- .../lwaftr/tests/propbased/genyang.lua | 82 +++++++++++++++---- 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index bd28029ec6..0b10c40d94 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -93,37 +93,66 @@ local function random_hexes() end -- return a random number, preferring boundary values -local function choose_range(lo, hi) +local function choose_bounded(lo, hi) local r = math.random() if r < 0.1 then local mid = math.ceil((hi + lo) / 2) - return choose({ 0, lo, lo + 1, mid, mid + 1, hi - 1, hi }) + return choose({ lo, lo + 1, mid, mid + 1, hi - 1, hi }) else return math.random(lo, hi) end end +-- choose a random number, taking range statements into account +local function choose_range(rng, lo, hi) + if #rng == 0 then + return choose_bounded(lo, hi) + elseif rng[1] == "or" then + local intervals = {} + local num_intervals = (#rng - 1) / 2 + + for i=1, num_intervals do + intervals[i] = { rng[2*i], rng[2*i+1] } + end + + return choose_range(choose(intervals), lo, hi) + else + local lo_rng, hi_rng = rng[1], rng[2] + + if lo_rng == "min" then + lo_rng = lo + end + if hi_rng == "max" then + hi_rng = hi + end + + return choose_bounded(math.max(lo_rng, lo), math.min(hi_rng, hi)) + end +end + local function value_from_type(a_type) local prim = a_type.primitive_type + local rng = a_type.range.value if prim == "int8" then - return choose_range(-128, 127) + return choose_range(rng, -128, 127) elseif prim == "int16" then - return choose_range(-32768, 32767) + return choose_range(rng, -32768, 32767) elseif prim == "int32" then - return choose_range(-2147483648, 2147483647) + return choose_range(rng, -2147483648, 2147483647) elseif prim == "int64" then - return choose_range(-9223372036854775809, 9223372036854775807) + return choose_range(rng, -9223372036854775809, 9223372036854775807) elseif prim == "uint8" then - return choose_range(0, 255) + return choose_range(rng, 0, 255) elseif prim == "uint16" then - return choose_range(0, 65535) + return choose_range(rng, 0, 65535) elseif prim == "uint32" then - return choose_range(0, 4294967295) + return choose_range(rng, 0, 4294967295) elseif prim == "uint64" then - return choose_range(0, 18446744073709551615) + return choose_range(rng, 0, 18446744073709551615) + -- TODO: account for fraction-digits and range elseif prim == "decimal64" then - local int64 = value_from_type({ primitive_type="int64" }) + local int64 = math.random(-9223372036854775809, 9223372036854775807) local exp = math.random(1, 18) -- see RFC 6020 sec 9.3.1 for lexical representation return string.format("%f", int64 * (10 ^ -exp)) @@ -132,15 +161,17 @@ local function value_from_type(a_type) elseif prim == "ipv4-address" then return math.random(0, 255) .. "." .. math.random(0, 255) .. "." .. math.random(0, 255) .. "." .. math.random(0, 255) - elseif prim == "ipv6-address" then + elseif prim == "ipv6-address" or prim == "ipv6-prefix" then local addr = random_hexes() for i=1, 7 do addr = addr .. ":" .. random_hexes() end + + if prim == "ipv6-prefix" then + return addr .. "/" .. math.random(0, 128) + end + return addr - elseif prim == "ipv6-prefix" then - local addr = value_from_type({ primitive_type = "ipv6-address" }) - return addr .. "/" .. math.random(0, 128) elseif prim == "mac-address" then local addr = random_hex() .. random_hex() for i=1,5 do @@ -398,15 +429,32 @@ function selftest() path.convert_path(grammar, generate_xpath(schema)) + -- check some int types with range statements + for i=1, 100 do + local val1 = value_from_type({ primitive_type="uint8", + range={ value = {1, 16} } }) + local val2 = value_from_type({ primitive_type="uint8", + range={ value = {"or", 1, 16, 18, 32} } }) + local val3 = value_from_type({ primitive_type="uint8", + range={ value = {"or", "min", 10, 250, "max"} } }) + assert(val1 >= 1 and val1 <= 16, string.format("test value: %d", val1)) + assert(val2 >= 1 and val2 <= 32 and val2 ~= 17, + string.format("test value: %d", val2)) + assert(val3 >= 0 and val3 <= 255 and not (val3 > 10 and val3 < 250), + string.format("test value: %d", val3)) + end + -- ensure decimal64 values match the right regexp for i=1, 100 do - local val = value_from_type({ primitive_type="decimal64" }) + local val = value_from_type({ primitive_type="decimal64", + range={ value={} } }) assert(string.match(val, "^-?%d+[.]%d+$"), string.format("test value: %s", val)) end -- ensure generated base64 values are decodeable for i=1, 100 do - local val = value_from_type({ primitive_type="binary" }) + local val = value_from_type({ primitive_type="binary", + range={ value={} }}) local cmd = string.format("echo \"%s\" | base64 -d > /dev/null", val) assert(os.execute(cmd) == 0, string.format("test value: %s", val)) end From c9e910f3d2d54f511a3e773262ecaf903b733fb0 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 13 Jan 2017 22:41:33 +0000 Subject: [PATCH 522/631] Validate PCI address for property based tests --- src/program/lwaftr/tests/propbased/common.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/program/lwaftr/tests/propbased/common.lua b/src/program/lwaftr/tests/propbased/common.lua index a5f7696a5b..34688242d6 100644 --- a/src/program/lwaftr/tests/propbased/common.lua +++ b/src/program/lwaftr/tests/propbased/common.lua @@ -3,6 +3,7 @@ module(..., package.seeall) -- common definitions for property-based tests for snabb config local S = require("syscall") +local pci = require("lib.hardware.pci") function make_handle_prop_args(name, duration, pidbox) local handler = function(prop_args) @@ -13,6 +14,8 @@ function make_handle_prop_args(name, duration, pidbox) -- TODO: validate the address local pci_addr = prop_args[1] + assert(S.stat(pci.path(pci_addr)), + string.format("Invalid PCI address: %s", pci_addr)) local pid = S.fork() if pid == 0 then From eaabeeaa4cfd60c97b399b1afebc6454ead16d57 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 13 Jan 2017 22:44:05 +0000 Subject: [PATCH 523/631] Don't fail if there isn't a range in the type --- src/program/lwaftr/tests/propbased/genyang.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 0b10c40d94..60b589e139 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -132,7 +132,13 @@ end local function value_from_type(a_type) local prim = a_type.primitive_type - local rng = a_type.range.value + local rng + + if a_type.range then + rng = a_type.range.value + else + rng = {} + end if prim == "int8" then return choose_range(rng, -128, 127) From 463a851b7d91a262633ff3708d292c9a879c968c Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Fri, 13 Jan 2017 23:43:14 +0000 Subject: [PATCH 524/631] Add code to occasionally ignore bounds --- src/program/lwaftr/tests/propbased/genyang.lua | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 60b589e139..a97073a279 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -92,10 +92,16 @@ local function random_hexes() return str end --- return a random number, preferring boundary values +-- return a random number, preferring boundary values and +-- sometimes returning results out of range local function choose_bounded(lo, hi) local r = math.random() - if r < 0.1 then + -- occasionally return values that are invalid for type + -- to provoke crashes + if r < 0.05 then + local off = math.random(1, 100) + return choose({ lo - off, hi + off }) + elseif r < 0.15 then local mid = math.ceil((hi + lo) / 2) return choose({ lo, lo + 1, mid, mid + 1, hi - 1, hi }) else @@ -105,7 +111,9 @@ end -- choose a random number, taking range statements into account local function choose_range(rng, lo, hi) - if #rng == 0 then + local r = math.random() + + if #rng == 0 or r < 0.1 then return choose_bounded(lo, hi) elseif rng[1] == "or" then local intervals = {} From 521bdd32bf74d9f307a4c6a26dc33cc4df4e1037 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Sun, 15 Jan 2017 02:21:37 +0000 Subject: [PATCH 525/631] Intentionally generate wrong types occasionally --- src/program/lwaftr/tests/propbased/genyang.lua | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index a97073a279..952156c27d 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -417,12 +417,28 @@ function generate_config_xpath(schema_name) return generate_xpath(schema, false), schema_name end +-- types that may be randomly picked for a fuzzed test case +local types = { "int8", "int16", "int32", "int64", "uint8", "uint16", + "uint32", "uint64", "decimal64", "boolean", "ipv4-address", + "ipv6-address", "ipv6-prefix", "mac-address", "string", + "binary" } + function generate_config_xpath_and_val(schema_name) if not schema_name then schema_name = choose(schemas) end local schema = schema.load_schema_by_name(schema_name) - local path, val = generate_xpath_and_val(schema) + local r = math.random() + local path, val + + -- once in a while, generate a nonsense value + if r < 0.05 then + path = generate_xpath(schema, false) + val = value_from_type({ primitive_type=choose(types) }) + else + path, val = generate_xpath_and_val(schema) + end + return path, val, schema_name end From aba11d0aec2f31790499223bc20ec3aecac5c98e Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Sun, 15 Jan 2017 02:38:23 +0000 Subject: [PATCH 526/631] Experiment with `snabb config add` testing Likely needs tweaking to be useful. The add command only works on arrays/tables, so it should work harder to generate the right paths for that. --- src/program/lwaftr/tests/propbased/genyang.lua | 12 ++++++++---- src/program/lwaftr/tests/propbased/prop_nocrash.lua | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 952156c27d..e546b9019c 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -10,16 +10,20 @@ require('lib.yang.schema').set_default_capabilities(capabilities) local schemas = { "ietf-softwire", "snabb-softwire-v1" } --- Generate a get/set command string given a pid string and optional schema -function generate_get_or_set(pid, schema) +-- Generate a get/set/add command string given a pid string and optional schema +function generate_get_set_add(pid, schema) local r = math.random() - if r > 0.5 then + if r <= 0.33 then local query, schema = generate_config_xpath(schema) return string.format("./snabb config get -s %s %s \"%s\"", schema, pid, query) - else + elseif r > 0.33 and r <= 0.66 then local query, val, schema = generate_config_xpath_and_val(schema) return string.format("./snabb config set -s %s %s \"%s\" \"%s\"", schema, pid, query, val) + else + local query, val, schema = generate_config_xpath_and_val(schema) + return string.format("./snabb config add -s %s %s \"%s\" \"%s\"", + schema, pid, query, val) end end diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index 57c3da673e..bc9dede9a7 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -7,7 +7,7 @@ local run_pid = {} local current_cmd function property() - current_cmd = genyang.generate_get_or_set(run_pid[1]) + current_cmd = genyang.generate_get_set_add(run_pid[1]) local results = (genyang.run_yang(current_cmd)) if common.check_crashed(results) then return false From 9520fa21f42ad2b3256c1ad99295ce9f51267ad2 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Mon, 16 Jan 2017 05:25:26 +0000 Subject: [PATCH 527/631] Generate `snabb config remove` commands too Similar to add, this needs to be more clever about picking array/table nodes --- .../lwaftr/tests/propbased/genyang.lua | 29 +++++++++++-------- .../lwaftr/tests/propbased/prop_nocrash.lua | 2 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index e546b9019c..26e2229a13 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -10,20 +10,31 @@ require('lib.yang.schema').set_default_capabilities(capabilities) local schemas = { "ietf-softwire", "snabb-softwire-v1" } --- Generate a get/set/add command string given a pid string and optional schema -function generate_get_set_add(pid, schema) - local r = math.random() - if r <= 0.33 then +-- choose an element of an array randomly +local function choose(choices) + local idx = math.random(#choices) + return choices[idx] +end + +-- Generate a get/set/add/remove string given a pid string and optional schema +function generate_any(pid, schema) + local cmd = choose({ "get", "add", "remove", "set" }) + + if cmd == "get" then local query, schema = generate_config_xpath(schema) return string.format("./snabb config get -s %s %s \"%s\"", schema, pid, query) - elseif r > 0.33 and r <= 0.66 then + elseif cmd == "set" then local query, val, schema = generate_config_xpath_and_val(schema) return string.format("./snabb config set -s %s %s \"%s\" \"%s\"", schema, pid, query, val) - else + elseif cmd == "add" then local query, val, schema = generate_config_xpath_and_val(schema) return string.format("./snabb config add -s %s %s \"%s\" \"%s\"", schema, pid, query, val) + else + local query, schema = generate_config_xpath(schema) + return string.format("./snabb config remove -s %s %s \"%s\"", + schema, pid, query) end end @@ -56,12 +67,6 @@ function run_yang(yang_cmd) return result end --- choose an element of an array randomly -local function choose(choices) - local idx = math.random(#choices) - return choices[idx] -end - -- choose a natural number (e.g., index or length of array) by -- repeating a cointoss local function choose_nat() diff --git a/src/program/lwaftr/tests/propbased/prop_nocrash.lua b/src/program/lwaftr/tests/propbased/prop_nocrash.lua index bc9dede9a7..71f8677bf3 100644 --- a/src/program/lwaftr/tests/propbased/prop_nocrash.lua +++ b/src/program/lwaftr/tests/propbased/prop_nocrash.lua @@ -7,7 +7,7 @@ local run_pid = {} local current_cmd function property() - current_cmd = genyang.generate_get_set_add(run_pid[1]) + current_cmd = genyang.generate_any(run_pid[1]) local results = (genyang.run_yang(current_cmd)) if common.check_crashed(results) then return false From f05dee67c4c0495b545abe8613a9a158f39586c6 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Tue, 17 Jan 2017 09:14:18 +0000 Subject: [PATCH 528/631] Fix selftest for genyang Don't generate values with invalid ranges or types for the selftest to ensure the test is predictable. --- src/program/lwaftr/tests/propbased/genyang.lua | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 26e2229a13..bb5b04b91a 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -10,6 +10,10 @@ require('lib.yang.schema').set_default_capabilities(capabilities) local schemas = { "ietf-softwire", "snabb-softwire-v1" } +-- toggles whether functions should intentionally generate invalid +-- values for fuzzing purposes +local generate_invalid = true + -- choose an element of an array randomly local function choose(choices) local idx = math.random(#choices) @@ -107,7 +111,7 @@ local function choose_bounded(lo, hi) local r = math.random() -- occasionally return values that are invalid for type -- to provoke crashes - if r < 0.05 then + if generate_invalid and r < 0.05 then local off = math.random(1, 100) return choose({ lo - off, hi + off }) elseif r < 0.15 then @@ -122,7 +126,7 @@ end local function choose_range(rng, lo, hi) local r = math.random() - if #rng == 0 or r < 0.1 then + if #rng == 0 or (generate_invalid and r < 0.1) then return choose_bounded(lo, hi) elseif rng[1] == "or" then local intervals = {} @@ -441,7 +445,7 @@ function generate_config_xpath_and_val(schema_name) local path, val -- once in a while, generate a nonsense value - if r < 0.05 then + if generate_invalid and r < 0.05 then path = generate_xpath(schema, false) val = value_from_type({ primitive_type=choose(types) }) else @@ -468,6 +472,9 @@ function selftest() path.convert_path(grammar, generate_xpath(schema)) + -- set flag to false to make tests predictable + generate_invalid = false + -- check some int types with range statements for i=1, 100 do local val1 = value_from_type({ primitive_type="uint8", From 4b0892c874dcd0a59e62c794194cefa0030ce00a Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Tue, 17 Jan 2017 21:12:50 +0000 Subject: [PATCH 529/631] Generate values/paths for config add/remove better This commit fixes both the query paths and generated values (for add) to test the add and remove commands more thoroughly. Also improves testing for the set command too. --- .../lwaftr/tests/propbased/genyang.lua | 126 +++++++++++------- 1 file changed, 80 insertions(+), 46 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index bb5b04b91a..6db6edd5c9 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -31,12 +31,30 @@ function generate_any(pid, schema) local query, val, schema = generate_config_xpath_and_val(schema) return string.format("./snabb config set -s %s %s \"%s\" \"%s\"", schema, pid, query, val) + -- use rejection sampling for add and remove commands to restrict to list or + -- leaf-list cases (for remove, we need a case with a selector too) + -- Note: this assumes a list or leaf-list case exists in the schema at all elseif cmd == "add" then - local query, val, schema = generate_config_xpath_and_val(schema) + local query, val, schema + local ok = false + while not ok do + query, val, schema = generate_config_xpath_and_val(schema) + if string.match(tostring(val), "^{.*}$") then + ok = true + end + end + --local query, val, schema = generate_config_xpath_and_val(schema) return string.format("./snabb config add -s %s %s \"%s\" \"%s\"", schema, pid, query, val) else - local query, schema = generate_config_xpath(schema) + local query, val, schema + local ok = false + while not ok do + query, val, schema = generate_config_xpath_and_val(schema) + if string.match(query, "[]]$") then + ok = true + end + end return string.format("./snabb config remove -s %s %s \"%s\"", schema, pid, query) end @@ -262,10 +280,13 @@ end -- from a config schema, generate an xpath query string -- this code is patterned off of the visitor used in lib.yang.data -local function generate_xpath_and_last_node(schema, for_state) +local function generate_xpath_and_node_info(schema, for_state) local path = "" local handlers = {} - local last_node + + -- data describing how to generate a value for the chosen path + -- it's a table with `node` and possibly-nil `selector` keys + local gen_info local function visit(node) local handler = handlers[node.kind] @@ -285,7 +306,7 @@ local function generate_xpath_and_last_node(schema, for_state) if id then visit(node.body[id]) else - last_node = node + gen_info = { node = node } end end function handlers.container(node) @@ -296,13 +317,17 @@ local function generate_xpath_and_last_node(schema, for_state) if math.random() < 0.9 then visit_body(node) else - last_node = node + gen_info = { node = node } end end handlers['leaf-list'] = function(node) - local selector = string.format("[position()=%d]", choose_nat()) + local idx = choose_nat() + local selector = string.format("[position()=%d]", idx) path = path .. "/" .. node.id .. selector - last_node = node + + -- TODO: this case should sometimes return the whole list instead of + -- just a single position in it + gen_info = { node = node, selector = idx } end function handlers.list(node) local key_types = {} @@ -311,7 +336,7 @@ local function generate_xpath_and_last_node(schema, for_state) path = path .. "/" .. node.id -- occasionally drop the selectors - if r < 0.9 then + if r < 0.7 then for key in (node.key):split(" +") do key_types[key] = node.body[key].type end @@ -320,18 +345,21 @@ local function generate_xpath_and_last_node(schema, for_state) local val = assert(value_from_type(type), type.primitive_type) path = path .. string.format("[%s=%s]", key, val) end - end - if math.random() < 0.9 then - visit_body(node) + -- continue path for child nodes + if math.random() < 0.5 then + visit_body(node) + else + gen_info = { node = node, selector = key_types } + end else - last_node = node + gen_info = { node = node } end end function handlers.leaf(node) path = path .. "/" .. node.id val = value_from_type(node.type) - last_node = node + gen_info = { node = node } end -- just produce "/" on rare occasions @@ -339,20 +367,27 @@ local function generate_xpath_and_last_node(schema, for_state) visit_body(schema) end - return path, last_node + return path, gen_info end -- similar to generating a query path like the function above, but -- generates a compound value for `snabb config set` at some schema -- node -local function generate_value_for_node(node) - local handlers = {} - - local function visit(node) - local handler = handlers[node.kind] - if handler then return handler(node) end +local function generate_value_for_node(gen_info) + -- hack for mutual recursion + local generate_compound + + local function generate(node) + if node.kind == "container" or node.kind == "list" then + return generate_compound(node) + elseif node.kind == "leaf-list" or node.kind == "leaf" then + return value_from_type(node.type) + end end - local function visit_body(node) + + -- take a node and (optional) keys and generate a compound value + -- the keys are only provided for a list node + generate_compound = function(node, keys) local ids = {} for id, node in pairs(node.body) do -- only choose nodes that are used in configs @@ -362,22 +397,24 @@ local function generate_value_for_node(node) end local val = "" + for _, id in ipairs(ids) do local subnode = node.body[id] local r = math.random() - if subnode.mandatory or r > 0.5 then + if (subnode.mandatory or r > 0.5) and + (not keys or not keys[id]) then if subnode.kind == "leaf-list" then local count = choose_nat() for i=0, count do - local subval = visit(subnode) + local subval = generate(subnode) val = val .. string.format("%s %s; ", id, subval) end elseif subnode.kind == "container" or subnode.kind == "list" then - local subval = visit(subnode) + local subval = generate(subnode) val = val .. string.format("%s {%s} ", id, subval) else - local subval = visit(subnode) + local subval = generate(subnode) val = val .. string.format("%s %s; ", id, subval) end end @@ -385,37 +422,34 @@ local function generate_value_for_node(node) return val end - function handlers.container(node) - return visit_body(node) - end - handlers['leaf-list'] = function(node) - return value_from_type(node.type) - end - function handlers.list(node) - -- FIXME: this will sometimes include a value for the list keys - -- which isn't valid when the query path sets the keys - return visit_body(node) - end - function handlers.leaf(node) - return value_from_type(node.type) - end - return visit(node) + local node = gen_info.node + if node.kind == "list" and gen_info.selector then + generate_compound(node, gen_info.selector) + else + -- a top-level list needs the brackets, e.g., as in + -- snabb config add /routes/route { addr 1.2.3.4; port 1; } + if node.kind == "list" then + return "{" .. generate(node) .. "}" + else + return generate(node) + end + end end local function generate_xpath(schema, for_state) - local path = generate_xpath_and_last_node(schema, for_state) + local path = generate_xpath_and_node_info(schema, for_state) return path end local function generate_xpath_and_val(schema) - local val, path, last + local val, path, gen_info while not val do - path, last = generate_xpath_and_last_node(schema) + path, gen_info = generate_xpath_and_node_info(schema) - if last then - val = generate_value_for_node(last) + if gen_info then + val = generate_value_for_node(gen_info) end end From 41a3bcb6f3a64c50d7ceb7aeb1c8554eb4566c67 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 18 Jan 2017 01:34:05 +0000 Subject: [PATCH 530/631] Adjust leaf-list case to not always use a selector --- src/program/lwaftr/tests/propbased/genyang.lua | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 6db6edd5c9..16d845cc09 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -321,13 +321,17 @@ local function generate_xpath_and_node_info(schema, for_state) end end handlers['leaf-list'] = function(node) - local idx = choose_nat() - local selector = string.format("[position()=%d]", idx) - path = path .. "/" .. node.id .. selector - - -- TODO: this case should sometimes return the whole list instead of - -- just a single position in it - gen_info = { node = node, selector = idx } + if math.random() < 0.7 then + local idx = choose_nat() + local selector = string.format("[position()=%d]", idx) + path = path .. "/" .. node.id .. selector + gen_info = { node = node, selector = idx } + -- sometimes omit the selector, for the benefit of commands + -- like add where a selector is not useful + else + path = path .. "/" .. node.id + gen_info = { node = node } + end end function handlers.list(node) local key_types = {} From 0b1417a8b15858e96f177ecb4e7ca01ea3df298f Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 18 Jan 2017 18:24:16 +0000 Subject: [PATCH 531/631] Tweak test iterations and timeout Having a large range of possible iterations makes the test less predictable, so reduced the range. --- src/program/lwaftr/tests/propbased/prop_sameval.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/prop_sameval.lua b/src/program/lwaftr/tests/propbased/prop_sameval.lua index 986827dcd8..96a0f98a12 100644 --- a/src/program/lwaftr/tests/propbased/prop_sameval.lua +++ b/src/program/lwaftr/tests/propbased/prop_sameval.lua @@ -18,7 +18,7 @@ function property() -- occasionally do a bunch of gets/sets at once if math.random() < 0.01 then - iters = math.random(100, 1000) + iters = math.random(100, 150) end for i=1, iters do @@ -66,6 +66,6 @@ function print_extra_information() end handle_prop_args = - common.make_handle_prop_args("prop_sameval", 60, run_pid) + common.make_handle_prop_args("prop_sameval", 90, run_pid) cleanup = common.make_cleanup(run_pid) From f4dae60e08c521c84c8b5f3beea54037d6e25c66 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 18 Jan 2017 20:10:50 +0000 Subject: [PATCH 532/631] Generate 64-bit ranges correctly The 64-bit cases generated in this commit ignore the `range` statements, however. In the current schemas, no uint64/int64 types are used with range statements. --- .../lwaftr/tests/propbased/genyang.lua | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index 16d845cc09..eb4b717b1d 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -3,6 +3,7 @@ module(..., package.seeall) -- This module provides functions for generating snabb config -- commands with random path queries and values +local ffi = require("ffi") local schema = require("lib.yang.schema") local capabilities = {['ietf-softwire']={feature={'binding', 'br'}}} @@ -123,6 +124,19 @@ local function random_hexes() return str end +-- generate a random 64-bit integer +local function random64() + local result = 0 + local r1 = ffi.cast("uint64_t", math.random(0, 2 ^ 32 - 1)) + local r2 = ffi.cast("uint64_t", math.random(0, 2 ^ 32 - 1)) + + for i=1, 32 do + r1 = r1 * 2ULL + end + + return r1 + r2 +end + -- return a random number, preferring boundary values and -- sometimes returning results out of range local function choose_bounded(lo, hi) @@ -186,7 +200,7 @@ local function value_from_type(a_type) elseif prim == "int32" then return choose_range(rng, -2147483648, 2147483647) elseif prim == "int64" then - return choose_range(rng, -9223372036854775809, 9223372036854775807) + return ffi.cast("int64_t", random64()) elseif prim == "uint8" then return choose_range(rng, 0, 255) elseif prim == "uint16" then @@ -194,13 +208,13 @@ local function value_from_type(a_type) elseif prim == "uint32" then return choose_range(rng, 0, 4294967295) elseif prim == "uint64" then - return choose_range(rng, 0, 18446744073709551615) + return random64() -- TODO: account for fraction-digits and range elseif prim == "decimal64" then - local int64 = math.random(-9223372036854775809, 9223372036854775807) + local int64 = ffi.cast("int64_t", random64()) local exp = math.random(1, 18) -- see RFC 6020 sec 9.3.1 for lexical representation - return string.format("%f", int64 * (10 ^ -exp)) + return string.format("%f", tonumber(int64 * (10 ^ -exp))) elseif prim == "boolean" then return choose({ true, false }) elseif prim == "ipv4-address" then From d2d3bd4fa7138dc087d39e7c689c5966d0fc6a2c Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 18 Jan 2017 20:26:50 +0000 Subject: [PATCH 533/631] Simplify the function in f4dae60e08c521c8 --- src/program/lwaftr/tests/propbased/genyang.lua | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/program/lwaftr/tests/propbased/genyang.lua b/src/program/lwaftr/tests/propbased/genyang.lua index eb4b717b1d..e2a532afd0 100644 --- a/src/program/lwaftr/tests/propbased/genyang.lua +++ b/src/program/lwaftr/tests/propbased/genyang.lua @@ -130,11 +130,7 @@ local function random64() local r1 = ffi.cast("uint64_t", math.random(0, 2 ^ 32 - 1)) local r2 = ffi.cast("uint64_t", math.random(0, 2 ^ 32 - 1)) - for i=1, 32 do - r1 = r1 * 2ULL - end - - return r1 + r2 + return r1 * 4294967296ULL + r2 end -- return a random number, preferring boundary values and From f8a09a198dca1dbc526cd40a100ca212edcfca80 Mon Sep 17 00:00:00 2001 From: Asumu Takikawa Date: Wed, 18 Jan 2017 21:09:19 +0000 Subject: [PATCH 534/631] Delete outdated TODO comment --- src/program/lwaftr/tests/propbased/common.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/src/program/lwaftr/tests/propbased/common.lua b/src/program/lwaftr/tests/propbased/common.lua index 34688242d6..3422cd6b7f 100644 --- a/src/program/lwaftr/tests/propbased/common.lua +++ b/src/program/lwaftr/tests/propbased/common.lua @@ -12,7 +12,6 @@ function make_handle_prop_args(name, duration, pidbox) os.exit(1) end - -- TODO: validate the address local pci_addr = prop_args[1] assert(S.stat(pci.path(pci_addr)), string.format("Invalid PCI address: %s", pci_addr)) From 64539d4868d4015f017e8f6695175758d85915b4 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 19 Jan 2017 16:36:03 +0100 Subject: [PATCH 535/631] Snabb lwAFTR v3.1.6 change log --- src/program/lwaftr/doc/CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index 6b1f7d75ca..ba85dc75ba 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,29 @@ # Change Log +## [3.1.6] - 2017-01-19 + +* Add basic error reporting to snabb-softwire-v1. + +* Add property-based testing for snabb config. + +* Add socket support for "snabb config listen". + +* Clean stale object files in program/lwaftr and program/snabbvmx. + +* Fix "lwaftr query". Added selftest. + +* Fix "snabb config remove" on arrays. + +* Fix bug parsing empty strings in YANG parser. + +* Fix tunnel-path-mtu and tunnel-payload-mtu in ietf-softwire. + +* Respond to ping packets to internal and external interfaces when + running in on-a-stick mode. Added test. + +* Several improvements in lwaftrctl script (no screen command, connect + via telnet, internet access in VM). + ## [3.1.5] - 2016-12-09 * Improve "snabb ps" output. Processes with a "*" by them are From fd46085bec53dfca20f571c90fb4e3730efd00cd Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 20 Jan 2017 13:37:01 +0000 Subject: [PATCH 536/631] Revert "Remove end-addr in psid-map" This reverts commit 86b9835eed86b0068d9e93637cc175167f1feba1. --- src/apps/lwaftr/binding_table.lua | 2 +- .../lwaftr/migrate_configuration/migrate_configuration.lua | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/apps/lwaftr/binding_table.lua b/src/apps/lwaftr/binding_table.lua index d8260454ce..99a252b3cc 100644 --- a/src/apps/lwaftr/binding_table.lua +++ b/src/apps/lwaftr/binding_table.lua @@ -260,7 +260,7 @@ function load(conf) 'psid_length '..psid_length..' + shift '..shift.. ' should not exceed 16') psid_value.psid_length, psid_value.shift = psid_length, shift - psid_builder:add_range(k.addr, k.addr, psid_value) + psid_builder:add_range(k.addr, v.end_addr or k.addr, psid_value) end local psid_map = psid_builder:build(psid_map_value_t()) return BindingTable.new(psid_map, conf.br_address, conf.softwire) diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index 7316841fab..d2062ecfaf 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -320,8 +320,10 @@ local function migrate_conf(old) local psid_map = cltable.new({ key_type = psid_key_t }) for addr, end_addr, params in old_bt.psid_map:iterate() do local reserved_ports_bit_count = 16 - params.psid_length - params.shift + if end_addr == addr then end_addr = nil end if reserved_ports_bit_count ~= 16 then psid_map[psid_key_t(addr)] = { + end_addr = end_addr, psid_length = params.psid_length, shift = params.shift, reserved_ports_bit_count = reserved_ports_bit_count From 7f44e9e89e8462a996e992647e86676c71c00f75 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 20 Jan 2017 16:48:16 +0100 Subject: [PATCH 537/631] Snabb lwAFTR v3.1.7 change log --- src/program/lwaftr/doc/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/program/lwaftr/doc/CHANGELOG.md b/src/program/lwaftr/doc/CHANGELOG.md index ba85dc75ba..6f920953b7 100644 --- a/src/program/lwaftr/doc/CHANGELOG.md +++ b/src/program/lwaftr/doc/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## [3.1.6] - 2017-01-20 + +* Reverts commit 86b9835 ("Remove end-addr in psid-map"), which + introduced a severe regression that caused high packet loss due + to not maching softwires. + ## [3.1.6] - 2017-01-19 * Add basic error reporting to snabb-softwire-v1. From d1ae27e2007461a6935fd9f5bd7678d9dcb69e23 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 25 Jan 2017 09:46:51 +0000 Subject: [PATCH 538/631] Retry ARP and NDP resolution indefinitely --- src/apps/lwaftr/ipv4_apps.lua | 10 +++------- src/apps/lwaftr/ipv6_apps.lua | 10 +++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/apps/lwaftr/ipv4_apps.lua b/src/apps/lwaftr/ipv4_apps.lua index 65b2ebb4af..0b9e51e46f 100644 --- a/src/apps/lwaftr/ipv4_apps.lua +++ b/src/apps/lwaftr/ipv4_apps.lua @@ -125,8 +125,6 @@ function ARP:new(conf) if not conf.dst_eth then o.arp_request_pkt = arp.form_request(conf.src_eth, conf.src_ipv4, conf.dst_ipv4) self.arp_request_interval = 3 -- Send a new arp_request every three seconds. - self.arp_request_max_retries = 5 -- Max number of arp_request retries. - self.arp_request_retries = 0 end o.dst_eth = conf.dst_eth -- intentionally nil if to request via ARP return o @@ -134,15 +132,11 @@ end function ARP:maybe_send_arp_request (output) if self.dst_eth then return end - if self.arp_request_retries == self.arp_request_max_retries then - error(("Could not resolve IPv4 address: %s"):format( - ipv4:ntop(self.conf.dst_ipv4))) - end self.next_arp_request_time = self.next_arp_request_time or engine.now() if self.next_arp_request_time <= engine.now() then + print(("ARP: Resolving '%s'"):format(ipv4:ntop(self.conf.dst_ipv4))) self:send_arp_request(output) self.next_arp_request_time = engine.now() + self.arp_request_interval - self.arp_request_retries = self.arp_request_retries + 1 end end @@ -162,6 +156,8 @@ function ARP:push() if not self.dst_eth and arp.is_arp_reply(p) then local dst_ethernet = arp.get_isat_ethernet(p) if dst_ethernet then + print(("ARP: '%s' resolved (%s)"):format(ipv4:ntop(self.conf.dst_ipv4), + ethernet:ntop(dst_ethernet))) self.dst_eth = dst_ethernet end packet.free(p) diff --git a/src/apps/lwaftr/ipv6_apps.lua b/src/apps/lwaftr/ipv6_apps.lua index e23b05dbed..a5538e0471 100644 --- a/src/apps/lwaftr/ipv6_apps.lua +++ b/src/apps/lwaftr/ipv6_apps.lua @@ -132,8 +132,6 @@ function NDP:new(conf) if not conf.dst_eth then self.ns_pkt = ndp.form_ns(conf.src_eth, conf.src_ipv6, conf.dst_ipv6) self.ns_interval = 3 -- Send a new NS every three seconds. - self.ns_max_retries = 5 -- Max number of NS retries. - self.ns_retries = 0 end o.dst_eth = conf.dst_eth -- Intentionally nil if to request by NS return o @@ -141,15 +139,11 @@ end function NDP:maybe_send_ns_request (output) if self.dst_eth then return end - if self.ns_retries == self.ns_max_retries then - error(("Could not resolve IPv6 address: %s"):format( - ipv6:ntop(self.conf.dst_ipv6))) - end self.next_ns_time = self.next_ns_time or engine.now() if self.next_ns_time <= engine.now() then + print(("NDP: Resolving '%s'"):format(ipv6:ntop(self.conf.dst_ipv6))) self:send_ns(output) self.next_ns_time = engine.now() + self.ns_interval - self.ns_retries = self.ns_retries + 1 end end @@ -172,6 +166,8 @@ function NDP:push() if not self.dst_eth and ndp.is_solicited_neighbor_advertisement(p) then local dst_ethernet = ndp.get_dst_ethernet(p, {self.conf.dst_ipv6}) if dst_ethernet then + print(("NDP: '%s' resolved (%s)"):format(ipv6:ntop(self.conf.dst_ipv6), + ethernet:ntop(dst_ethernet))) self.dst_eth = dst_ethernet end packet.free(p) From d4efeb785de2ae8bdf04f81435cc22ad7842acb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 1 Feb 2017 10:17:56 +0100 Subject: [PATCH 539/631] lwaftrctl: allow setting QEMU_ARGS --- src/program/lwaftr/virt/lwaftrctl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index 9ef11704c3..c2d32988d8 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -23,7 +23,6 @@ # The command 'lwaftrctl all start', run all the steps above. QEMU=qemu-system-x86_64 -QEMU_ARGS="" LWAFTR_DELAY=${LWAFTR_DELAY:-"20"} tmux_session="" From b9d1c93023039033e79646f79a8c190e0fcb6dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 1 Feb 2017 10:18:28 +0100 Subject: [PATCH 540/631] lwaftrctl: fix docs --- src/program/lwaftr/virt/lwaftrctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index c2d32988d8..27fb087b02 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -238,7 +238,7 @@ function restart_command { function usage { local exit_code=$1 - echo "Usage: lwaftrctl -f lwaftrctl.conf [start|stop|restart] [all|snabbnfv|vm]" + echo "Usage: lwaftrctl -f lwaftrctl.conf [start|stop|restart] [all|snabbnfv|vm|lwaftr]" exit $exit_code } From 13ed45f875a5ed3f3782acf8fef25ec7320dc3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 1 Feb 2017 10:18:49 +0100 Subject: [PATCH 541/631] lwaftr: replace tap interface with 9p share - guest logs (kernel dmesg) are now displayed on stdout (as in upstream snabb bench) - allow to pass extra kernel parameters --- src/program/lwaftr/virt/lwaftrctl | 7 ++++--- src/program/lwaftr/virt/lwaftrctl.conf.example | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/virt/lwaftrctl b/src/program/lwaftr/virt/lwaftrctl index 27fb087b02..f92d699ea6 100755 --- a/src/program/lwaftr/virt/lwaftrctl +++ b/src/program/lwaftr/virt/lwaftrctl @@ -48,16 +48,17 @@ function qemu_start_vm { "sudo numactl -m ${NUMA_NODE} taskset -c ${QEMU_CORE} \ $QEMU $QEMU_ARGS \ -kernel ${BZ_IMAGE} \ - -append \"earlyprintk root=/dev/vda rw console=tty0 ip=${VM_IP} hugepages=256\" \ + -append \"earlyprintk root=/dev/vda rw console=ttyS1 ${KERNEL_PARAMS} ip=${VM_IP} hugepages=256\" \ -M pc -smp 1 -cpu host -m ${MEM} -enable-kvm \ + -fsdev local,security_model=passthrough,id=fsdev0,path=${SHARED_LOCATION} \ + -device virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=share \ -numa node,memdev=mem -object memory-backend-file,id=mem,size=${MEM},mem-path=${HUGEPAGES_FS},share=on \ -netdev type=vhost-user,id=net0,chardev=char0 -chardev socket,id=char0,path=${SNABBNFV_SOCK1},server \ -device virtio-net-pci,netdev=net0,addr=0x8,mac=${SNABBNFV_MAC1} \ -netdev type=vhost-user,id=net1,chardev=char1 -chardev socket,id=char1,path=${SNABBNFV_SOCK2},server \ -device virtio-net-pci,netdev=net1,addr=0x9,mac=${SNABBNFV_MAC2} \ - -netdev type=tap,id=net2,ifname=${IFACE},vhost=on,script=${NETSCRIPT} \ - -device virtio-net-pci,netdev=net2,addr=0xa,mac=52:54:00:00:00:03 \ -serial telnet:localhost:${VM_PORT},server,nowait \ + -serial stdio \ -display none \ -drive if=virtio,format=${VM_FORMAT},file=${VM_IMAGE} " \ "qemu0.log" diff --git a/src/program/lwaftr/virt/lwaftrctl.conf.example b/src/program/lwaftr/virt/lwaftrctl.conf.example index a5bfec8052..ecfca64805 100644 --- a/src/program/lwaftr/virt/lwaftrctl.conf.example +++ b/src/program/lwaftr/virt/lwaftrctl.conf.example @@ -15,6 +15,7 @@ QEMU_CORE=0 MEM=2G HUGEPAGES_FS=/dev/hugepages +KERNEL_PARAMS= BZ_IMAGE=~/.test_env/bzImage VM_IMAGE=~/.test_env/lwaftr-vm.img @@ -25,6 +26,8 @@ VM_IP=10.21.21.2 IFACE=mgmt0 NETSCRIPT=$SNABB_LWAFTR/virt/setup_networks/lwaftr1.sh +SHARED_LOCATION= + # SnabbNFV SNABBNFV_CORE1=1 From 95e2a661b3d2a803c896c44809caeb902138ded4 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 6 Feb 2017 12:27:38 +0100 Subject: [PATCH 542/631] Added a regression test for end-addr This test checks that all softwires within a range specified with end-addr actually work. Solves https://github.com/Igalia/snabb/issues/723 . --- .../data/counters/regressiontest-endaddr.lua | 10 +++ .../lwaftr/tests/data/icmp_endaddr.conf | 63 ++++++++++++++++++ .../data/regressiontest-endaddr-v4-input.pcap | Bin 0 -> 6064 bytes .../regressiontest-endaddr-v6-output.pcap | Bin 0 -> 6224 bytes .../lwaftr/tests/end-to-end/test_env.sh | 4 ++ 5 files changed, 77 insertions(+) create mode 100644 src/program/lwaftr/tests/data/counters/regressiontest-endaddr.lua create mode 100644 src/program/lwaftr/tests/data/icmp_endaddr.conf create mode 100644 src/program/lwaftr/tests/data/regressiontest-endaddr-v4-input.pcap create mode 100644 src/program/lwaftr/tests/data/regressiontest-endaddr-v6-output.pcap diff --git a/src/program/lwaftr/tests/data/counters/regressiontest-endaddr.lua b/src/program/lwaftr/tests/data/counters/regressiontest-endaddr.lua new file mode 100644 index 0000000000..b06cf00002 --- /dev/null +++ b/src/program/lwaftr/tests/data/counters/regressiontest-endaddr.lua @@ -0,0 +1,10 @@ +return { + ["in-ipv4-bytes"] = 5976, + ["in-ipv4-frag-reassembly-unneeded"] = 4, + ["in-ipv4-packets"] = 4, + ["memuse-ipv4-frag-reassembly-buffer"] = 463571780, + ["memuse-ipv6-frag-reassembly-buffer"] = 464727376, + ["out-ipv6-bytes"] = 6136, + ["out-ipv6-frag-not"] = 4, + ["out-ipv6-packets"] = 4, +} diff --git a/src/program/lwaftr/tests/data/icmp_endaddr.conf b/src/program/lwaftr/tests/data/icmp_endaddr.conf new file mode 100644 index 0000000000..ddcfdb9e77 --- /dev/null +++ b/src/program/lwaftr/tests/data/icmp_endaddr.conf @@ -0,0 +1,63 @@ +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.233; + end-addr 178.79.150.236; + psid-length 0; + shift 16; + } + softwire { + ipv4 178.79.150.233; + psid 0; + b4-ipv6 127:10:20:30:30:50:60:128; + } + softwire { + ipv4 178.79.150.234; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.235; + psid 0; + b4-ipv6 127:10:20:30:50:50:60:128; + } + softwire { + ipv4 178.79.150.236; + psid 0; + b4-ipv6 127:10:20:30:60:50:60:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 2000; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 2000; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + } +} diff --git a/src/program/lwaftr/tests/data/regressiontest-endaddr-v4-input.pcap b/src/program/lwaftr/tests/data/regressiontest-endaddr-v4-input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..457141907c9b6e73d554a9adfa6fd1038356873c GIT binary patch literal 6064 zcmca|c+)~A1{MYw_+QV!zzF1|9h(sG^$8opHC6^NW@TY!0*W&-F|)98Ft{?Xo&YIx zU{n7H658ZH?Ip(_t{#{v0Za-EYHJu63=EBoO-#+qN9oaU7)=MG`Czmh7+mE5sFdIN zf{0T71GJQXg;C0{9xdfZQ}y6V)uZt-8Xu#r4Mb}LRLVboO++dG9$L!3#wg`ijh6DG zrOIfjGFqx2N|n+4Fu3vqsFYvyj)+qJ9ki5xgHg(_94+NXQ}y6V)uZt-8Xu#zE~3^2 E04aFEdH?_b literal 0 HcmV?d00001 diff --git a/src/program/lwaftr/tests/data/regressiontest-endaddr-v6-output.pcap b/src/program/lwaftr/tests/data/regressiontest-endaddr-v6-output.pcap new file mode 100644 index 0000000000000000000000000000000000000000..e14bcdc52209096a1eec5af5351eba6a276063dd GIT binary patch literal 6224 zcmca|c+)~A1{MYw`2U}Qff2?5(*IZ?*aZxfz@Y7J0#Jta1j~O04hBvJE(UG}9tK_p zJ_de9bp`etqopb)CQ|ZYlG3$Ke$r=Xnc&u z$7rhr(JG;^HVD9|4c Date: Mon, 6 Feb 2017 16:29:51 +0100 Subject: [PATCH 543/631] #700 - Allow for missing optional containers snabb set This allows you to miss out optional arguments where there is a container without any required leaves in it. Previous you would often get a casting error when they were left empty of omitted. --- src/lib/yang/data.lua | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 8d71a7bd7d..3586fcaf43 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -204,9 +204,16 @@ local function struct_parser(keyword, members, ctype) return parse1(P) end local struct_t = ctype and typeof(ctype) - local function finish(out) - -- FIXME check mandatory values. - if struct_t then return struct_t(out) else return out end + local function finish(out, leaf) + -- FIXME check mandatory values. + if struct_t then + local ret + if out == nil then ret = struct_t() + else ret = struct_t(out) end + return ret + else + return out + end end return {init=init, parse=parse, finish=finish} end From 6a4cd40b18702692aa67416206b784e9bf7cfb4c Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Tue, 7 Feb 2017 11:02:09 +0100 Subject: [PATCH 544/631] Added vlan version of tests --- src/program/lwaftr/tests/data/add-vlan.sh | 2 + .../lwaftr/tests/data/vlan/icmp_endaddr.conf | 65 ++++++++++++++++++ .../vlan/regressiontest-endaddr-v4-input.pcap | Bin 0 -> 6080 bytes .../regressiontest-endaddr-v6-output.pcap | Bin 0 -> 6240 bytes 4 files changed, 67 insertions(+) create mode 100644 src/program/lwaftr/tests/data/vlan/icmp_endaddr.conf create mode 100644 src/program/lwaftr/tests/data/vlan/regressiontest-endaddr-v4-input.pcap create mode 100644 src/program/lwaftr/tests/data/vlan/regressiontest-endaddr-v6-output.pcap diff --git a/src/program/lwaftr/tests/data/add-vlan.sh b/src/program/lwaftr/tests/data/add-vlan.sh index 50a5c5745c..0e9fca2f43 100755 --- a/src/program/lwaftr/tests/data/add-vlan.sh +++ b/src/program/lwaftr/tests/data/add-vlan.sh @@ -24,6 +24,7 @@ V4=( incoming-icmpv4-echo-request-invalid-icmp-checksum.pcap incoming-icmpv4-echo-request-unbound.pcap incoming-icmpv4-echo-request.pcap + regressiontest-endaddr-v4-input.pcap response-ipv4-icmp31-inet.pcap response-ipv4-icmp34-inet.pcap tcp-afteraftr-ipv6-wrongiface.pcap @@ -79,6 +80,7 @@ V6=( recap-ipv6.pcap recap-ipv6-n64.pcap recap-tocustom-BRIP-ipv6.pcap + regressiontest-endaddr-v6-output.pcap regressiontest-signedntohl-frags.pcap regressiontest-signedntohl-frags-output.pcap response-ipv6-tunneled-icmpv4_31-tob4.pcap diff --git a/src/program/lwaftr/tests/data/vlan/icmp_endaddr.conf b/src/program/lwaftr/tests/data/vlan/icmp_endaddr.conf new file mode 100644 index 0000000000..5448e49da9 --- /dev/null +++ b/src/program/lwaftr/tests/data/vlan/icmp_endaddr.conf @@ -0,0 +1,65 @@ +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.233; + end-addr 178.79.150.236; + psid-length 0; + shift 16; + } + softwire { + ipv4 178.79.150.233; + psid 0; + b4-ipv6 127:10:20:30:30:50:60:128; + } + softwire { + ipv4 178.79.150.234; + psid 0; + b4-ipv6 127:10:20:30:40:50:60:128; + } + softwire { + ipv4 178.79.150.235; + psid 0; + b4-ipv6 127:10:20:30:50:50:60:128; + } + softwire { + ipv4 178.79.150.236; + psid 0; + b4-ipv6 127:10:20:30:60:50:60:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + mtu 2000; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1092; + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + mtu 2000; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + vlan-tag 1638; + } +} diff --git a/src/program/lwaftr/tests/data/vlan/regressiontest-endaddr-v4-input.pcap b/src/program/lwaftr/tests/data/vlan/regressiontest-endaddr-v4-input.pcap new file mode 100644 index 0000000000000000000000000000000000000000..1cad589423ece7855a0b0366f2e34725f61a6c08 GIT binary patch literal 6080 zcmca|c+)~A1{MYw`2U}Qff2|_J2oNW>k~GHTdWLV%*w*d1QcgvVrF4&WMFaOU~pw% zJpoecz^48YB)rLg+DncrtHC$vPa`%G(JXa IUPR3c00Ec9J^%m! literal 0 HcmV?d00001 diff --git a/src/program/lwaftr/tests/data/vlan/regressiontest-endaddr-v6-output.pcap b/src/program/lwaftr/tests/data/vlan/regressiontest-endaddr-v6-output.pcap new file mode 100644 index 0000000000000000000000000000000000000000..f96da1926650f0ea0c46c17857c4a60de6bb00fe GIT binary patch literal 6240 zcmca|c+)~A1{MYw`2U}Qff2?5(oAd+>;eW#V9?0GmezJR0VvCQg5^I02LmSq7Xvo~ z4+Ad)9|J$5I)ea%0)qh%1~4QrYPbSbfsC|g)A-29#LTkEf7(lqKU_U9O9Ge_7}VA< zFc=sb8Jn1znUB(=;V_yGM)SdFIWV}&0a$rWL5<*mQ6qeS)(EdKYJ}CJHNt2P7+g7E zG(JY-W3*+0Xqix Date: Tue, 7 Feb 2017 12:21:26 +0100 Subject: [PATCH 545/631] Remove busywait from main process when leader (#732) --- src/program/lwaftr/run/run.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 8b6cb78ee2..01e56558ad 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -217,7 +217,10 @@ function run(args) end end - engine.busywait = true + if not opts.reconfigurable then + -- The leader does not need all the CPU, only the followers do. + engine.busywait = true + end if opts.duration then engine.main({duration=opts.duration, report={showlinks=true}}) else From aa496be083666aef18d685736ea9af5535ef1d54 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Wed, 8 Feb 2017 12:55:26 +0100 Subject: [PATCH 546/631] Automate creation of tarball releases with Hydra (#733) --- tarball.nix | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tarball.nix diff --git a/tarball.nix b/tarball.nix new file mode 100644 index 0000000000..ad870e5a33 --- /dev/null +++ b/tarball.nix @@ -0,0 +1,66 @@ +# Run like this: +# nix-build ./tarball.nix +# or +# nix-build --argstr hydraName snabb-lwaftr --argstr version v1.0.0 ./tarball.nix +# and the release tarball will be written to ./result/ . It will contain +# both the sources and the executable, patched to run on Linux LSB systems. +# +# NOTE: Hydra doesn't use the --tags parameter in the "git describe" command (see +# https://github.com/NixOS/hydra/blob/master/src/lib/Hydra/Plugin/GitInput.pm#L148 +# ), so lightweight (non-annotated) tags are not found. Always create annotated +# tags with the "git tag" command by using one of the -a, -s or -u options. + +{ nixpkgs ? +, hydraName ? "snabb" +, src ? ./. +, version ? "dev" +}: + +let + pkgs = import nixpkgs {}; + name = (src: + if builtins.isAttrs src then + # Called from Hydra with "src" as "Git version", get the version from git tags. + # If the last commit is the tag's one, you'll just get the tag name: "v3.1.7"; + # otherwise you'll also get the number of commits since the last tag, and the + # shortened commit checksum: "v3.1.7-7-g89747a1". + "${hydraName}-${src.gitTag}" + else + # Called from the command line, the user supplies the version. + "${hydraName}-${version}") src; +in { + tarball = pkgs.stdenv.mkDerivation rec { + inherit name src; + + buildInputs = with pkgs; [ makeWrapper patchelf ]; + + postUnpack = '' + mkdir -p $out/$name + cp -a $sourceRoot/* $out/$name + ''; + + preBuild = '' + make clean + ''; + + installPhase = '' + mv src/snabb $out + ''; + + fixupPhase = '' + patchelf --shrink-rpath $out/snabb + patchelf --set-rpath /lib/x86_64-linux-gnu $out/snabb + patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 $out/snabb + ''; + + doDist = true; + + distPhase = '' + cd $out + tar Jcf $name.tar.xz * + # Make the tarball available for download through Hydra. + mkdir -p $out/nix-support + echo "file tarball $out/$name.tar.xz" >> $out/nix-support/hydra-build-products + ''; + }; +} From 074adda2353b488a786f5ccda9cd7b659974ded4 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 13 Feb 2017 14:38:32 +0100 Subject: [PATCH 547/631] Refactor query tests to use common test tooling (#741) --- src/program/lwaftr/query/selftest.sh | 10 ++- src/program/lwaftr/query/tests/test_query.sh | 22 +++--- .../query/tests/test_query_reconfigurable.sh | 22 +++--- src/program/lwaftr/tests/common.sh | 71 +++++++++++++++++++ 4 files changed, 101 insertions(+), 24 deletions(-) create mode 100755 src/program/lwaftr/tests/common.sh diff --git a/src/program/lwaftr/query/selftest.sh b/src/program/lwaftr/query/selftest.sh index 0d884555d9..825183f47a 100755 --- a/src/program/lwaftr/query/selftest.sh +++ b/src/program/lwaftr/query/selftest.sh @@ -1,4 +1,10 @@ #!/usr/bin/env bash -./program/lwaftr/query/tests/test_query_reconfigurable.sh -./program/lwaftr/query/tests/test_query.sh +LWAFTR_DIR="./program/lwaftr" + +# Pass TEST_DIR and QUERY_TEST_DIR to the invoked scripts. +export TEST_DIR="${LWAFTR_DIR}/tests" +export QUERY_TEST_DIR="${LWAFTR_DIR}/query/tests" + +${QUERY_TEST_DIR}/test_query.sh +${QUERY_TEST_DIR}/test_query_reconfigurable.sh diff --git a/src/program/lwaftr/query/tests/test_query.sh b/src/program/lwaftr/query/tests/test_query.sh index c3df4bc49a..bbbb283311 100755 --- a/src/program/lwaftr/query/tests/test_query.sh +++ b/src/program/lwaftr/query/tests/test_query.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash -SKIPPED_CODE=43 +TEST_NAME="query" -if [[ -z "$SNABB_PCI0" ]]; then - echo "SNABB_PCI0 not set" - exit $SKIPPED_CODE -fi +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh -if [[ -z "$SNABB_PCI1" ]]; then - echo "SNABB_PCI1 not set" - exit $SKIPPED_CODE -fi +check_for_root +check_nics_available $TEST_NAME + +# QUERY_TEST_DIR is also set by the caller. +source ${QUERY_TEST_DIR}/test_env.sh -source ./program/lwaftr/query/tests/test_env.sh +echo "Testing ${TEST_NAME}" trap cleanup EXIT HUP INT QUIT TERM -LWAFTR_CONF=./program/lwaftr/tests/data/no_icmp.conf LWAFTR_NAME=lwaftr-$$ +LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf # Run lwAFTR. tmux_launch "lwaftr" "./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" "lwaftr.log" diff --git a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh index f28ed4d9e6..484bb89709 100755 --- a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh +++ b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash -SKIPPED_CODE=43 +TEST_NAME="query-reconfigurable" -if [[ -z "$SNABB_PCI0" ]]; then - echo "SNABB_PCI0 not set" - exit $SKIPPED_CODE -fi +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh -if [[ -z "$SNABB_PCI1" ]]; then - echo "SNABB_PCI1 not set" - exit $SKIPPED_CODE -fi +check_for_root +check_nics_available $TEST_NAME + +# QUERY_TEST_DIR is also set by the caller. +source ${QUERY_TEST_DIR}/test_env.sh -source ./program/lwaftr/query/tests/test_env.sh +echo "Testing ${TEST_NAME}" trap cleanup EXIT HUP INT QUIT TERM -LWAFTR_CONF=./program/lwaftr/tests/data/no_icmp.conf LWAFTR_NAME=lwaftr-$$ +LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf # Run lwAFTR. tmux_launch "lwaftr" "./snabb lwaftr run --reconfigurable --name $LWAFTR_NAME --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" "lwaftr.log" diff --git a/src/program/lwaftr/tests/common.sh b/src/program/lwaftr/tests/common.sh new file mode 100755 index 0000000000..e832343569 --- /dev/null +++ b/src/program/lwaftr/tests/common.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +SKIPPED_CODE=43 + +# Show $1 as error message and exit with code $2, or 1 if not passed. +function exit_on_error { + (>&2 echo $1) + if [[ -n "$2" ]]; then + exit $2 + else + exit 1 + fi +} + +# Check that the script is run as root, otherwise exit. +function check_for_root { + if [[ $EUID != 0 ]]; then + exit_on_error "This script must be run as root" + fi +} + +# Check that a command is available, otherwise exit with code $SKIPPED_CODE. +function check_command_available { + which "$1" &> /dev/null + if [[ $? -ne 0 ]]; then + exit_on_error "No $1 tool present, unable to run test." $SKIPPED_CODE + fi +} + +# Check that NIC interfaces are available, otherwise exit with code $SKIPPED_CODE. +function check_nics_available { + if [[ -z "$SNABB_PCI0" ]]; then + exit_on_error "SNABB_PCI0 not set, skipping $1" $SKIPPED_CODE + fi + if [[ -z "$SNABB_PCI1" ]]; then + exit_on_error "SNABB_PCI1 not set, skipping $1" $SKIPPED_CODE + fi +} + +# Check that a file exists, otherwise exit. +# If the second argument is "--remove", remove the file. +function assert_file_exists { + if [[ ! -f "$1" ]]; then + exit_on_error "File $1 does not exists." + fi + if [[ "$2" == "--remove" ]]; then + rm -f "$1" + fi +} + +# Check equality of the first two arguments. +# The third argument will be displayed if the check fails. +# e.g. +# $ assert_equal "yellow "cat" -> error +# $ assert_equal "yellow "cat" "Cat not yellow" -> error with message +# $ assert_equal "banana" "banana" -> nothing (valid) +function assert_equal { + if [[ -z "$2" ]]; then + exit_on_error "assert_equal: not enough arguments." + exit 1 + fi + if [[ "$1" == "$2" ]]; then + return + else + if [[ -z "$3" ]]; then + exit_on_error "Error: $1 != $2" + else + exit_on_error "Error: $3" + fi + fi +} From 68ec9a279f0cd6e286bfbdf17ea2fd393fbda166 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 13 Feb 2017 16:19:50 +0100 Subject: [PATCH 548/631] Rework config tests to use common test tooling (#740) --- src/program/lwaftr/tests/config/selftest.sh | 16 +++-- .../lwaftr/tests/config/test-config-add.sh | 28 ++++---- .../tests/config/test-config-get-state.sh | 36 +++++----- .../lwaftr/tests/config/test-config-get.sh | 26 +++---- .../lwaftr/tests/config/test-config-listen.sh | 68 +++++++++---------- .../lwaftr/tests/config/test-config-remove.sh | 30 ++++---- .../lwaftr/tests/config/test-config-set.sh | 32 ++++----- src/program/lwaftr/tests/config/test_env.sh | 42 ++++++++++++ src/program/lwaftr/tests/config/tools.sh | 66 ------------------ 9 files changed, 159 insertions(+), 185 deletions(-) create mode 100755 src/program/lwaftr/tests/config/test_env.sh delete mode 100755 src/program/lwaftr/tests/config/tools.sh diff --git a/src/program/lwaftr/tests/config/selftest.sh b/src/program/lwaftr/tests/config/selftest.sh index 817410f78c..8f9d9d2c74 100755 --- a/src/program/lwaftr/tests/config/selftest.sh +++ b/src/program/lwaftr/tests/config/selftest.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash -set -o errexit +# Pass TEST_DIR and CONFIG_TEST_DIR to the invoked scripts. +export TEST_DIR="./program/lwaftr/tests" +export CONFIG_TEST_DIR=${TEST_DIR}/config -./program/lwaftr/tests/config/test-config-get.sh -./program/lwaftr/tests/config/test-config-set.sh -./program/lwaftr/tests/config/test-config-add.sh -./program/lwaftr/tests/config/test-config-remove.sh -./program/lwaftr/tests/config/test-config-get-state.sh -./program/lwaftr/tests/config/test-config-listen.sh +${CONFIG_TEST_DIR}/test-config-get.sh +${CONFIG_TEST_DIR}/test-config-set.sh +${CONFIG_TEST_DIR}/test-config-add.sh +${CONFIG_TEST_DIR}/test-config-remove.sh +${CONFIG_TEST_DIR}/test-config-get-state.sh +${CONFIG_TEST_DIR}/test-config-listen.sh diff --git a/src/program/lwaftr/tests/config/test-config-add.sh b/src/program/lwaftr/tests/config/test-config-add.sh index b1ff8153df..d0c154466d 100755 --- a/src/program/lwaftr/tests/config/test-config-add.sh +++ b/src/program/lwaftr/tests/config/test-config-add.sh @@ -2,18 +2,18 @@ ## This adds a softwire section and then checks it can be got ## back and that all the values are as they should be. -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" 1>&2 - exit 1 -fi - -# Load the tools to be able to test stuff. -BASEDIR="`pwd`" -cd "`dirname \"$0\"`" -source tools.sh -cd $BASEDIR - -# Come up with a name for the lwaftr +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh + +check_for_root + +# CONFIG_TEST_DIR is also set by the caller. +source ${CONFIG_TEST_DIR}/test_env.sh + +echo "Testing config add" + +# Come up with a name for the lwaftr. SNABB_NAME="`random_name`" # Start the bench command. @@ -23,11 +23,11 @@ start_lwaftr_bench $SNABB_NAME TEST_SOFTWIRE="{ ipv4 1.2.3.4; psid 72; b4-ipv6 ::1; br 1; }" ./snabb config add "$SNABB_NAME" "/softwire-config/binding-table/softwire" "$TEST_SOFTWIRE" -# Check it can get this just fine +# Check it can get this just fine. ./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=1.2.3.4][psid=72] &> /dev/null assert_equal $? 0 -# Test that the b4-ipv4 is correct +# Test that the b4-ipv4 is correct. ADDED_B4_IPV4="`./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=1.2.3.4][psid=72]/b4-ipv6`" assert_equal "$ADDED_B4_IPV4" "::1" diff --git a/src/program/lwaftr/tests/config/test-config-get-state.sh b/src/program/lwaftr/tests/config/test-config-get-state.sh index c256c988f3..7e3fc7180c 100755 --- a/src/program/lwaftr/tests/config/test-config-get-state.sh +++ b/src/program/lwaftr/tests/config/test-config-get-state.sh @@ -1,37 +1,37 @@ #!/usr/bin/env bash -## This makes verious quries to snabb config get-state to try and verify +## This makes various queries to snabb config get-state to verify ## that it will run and produce values. The script has no way of -## validating the acuracy of the values but it'll check it works. +## validating the accuracy of the values, but it'll check it works. -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" 1>&2 - exit 1 -fi +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh + +check_for_root + +# CONFIG_TEST_DIR is also set by the caller. +source ${CONFIG_TEST_DIR}/test_env.sh -# Load the tools to be able to test stuff. -BASEDIR="`pwd`" -cd "`dirname \"$0\"`" -source tools.sh -cd $BASEDIR +echo "Testing config get state" -# Come up with a name for the lwaftr +# Come up with a name for the lwaftr. SNABB_NAME="`random_name`" # Start the bench command. start_lwaftr_bench $SNABB_NAME -# Selecting a few at random which should have non-zero results -IN_IPV4="`snabb config get-state $SNABB_NAME /softwire-state/in-ipv4-bytes`" +# Select a few at random which should have non-zero results. +IN_IPV4="`./snabb config get-state $SNABB_NAME /softwire-state/in-ipv4-bytes`" if [[ "$IN_IPV4" == "0" ]]; then - produce_error "Counter should not show zero." + exit_on_error "Counter should not show zero." fi -OUT_IPV4="`snabb config get-state $SNABB_NAME /softwire-state/out-ipv4-bytes`" +OUT_IPV4="`./snabb config get-state $SNABB_NAME /softwire-state/out-ipv4-bytes`" if [[ "$IN_IPV4" == "0" ]]; then - produce_error "Counter should not show zero." + exit_on_error "Counter should not show zero." fi -./snabb config get-state "$SNABB_NAME" / +./snabb config get-state "$SNABB_NAME" / > /dev/null assert_equal "$?" "0" stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-get.sh b/src/program/lwaftr/tests/config/test-config-get.sh index 64bc6394ab..7c0259b2da 100755 --- a/src/program/lwaftr/tests/config/test-config-get.sh +++ b/src/program/lwaftr/tests/config/test-config-get.sh @@ -1,21 +1,21 @@ #!/usr/bin/env bash -## This tests querying from a known config, the test is obviously +## This tests querying from a known config. The test is obviously ## dependent on the values in the test data files used, however this -## allows for testing basic "getting". It performs numerous gets which +## allows for testing basic "getting". It performs numerous gets ## on different paths. -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" 1>&2 - exit 1 -fi +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh -# Load the tools to be able to test stuff. -BASEDIR="`pwd`" -cd "`dirname \"$0\"`" -source tools.sh -cd $BASEDIR +check_for_root -# Come up with a name for the lwaftr +# CONFIG_TEST_DIR is also set by the caller. +source ${CONFIG_TEST_DIR}/test_env.sh + +echo "Testing config get" + +# Come up with a name for the lwaftr. SNABB_NAME="`random_name`" # Start the bench command. @@ -31,7 +31,7 @@ assert_equal "$EXTERNAL_IP" "10.10.10.10" BT_B4_IPV6="`./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=178.79.150.233][psid=7850]/b4-ipv6`" assert_equal "$BT_B4_IPV6" "127:11:12:13:14:15:16:128" -# Finally test getting a value from the ietf-softwire schema +# Finally test getting a value from the ietf-softwire schema. IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=127:22:33:44:55:66:77:128]/binding-ipv4-addr" BINDING_IPV4="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" assert_equal "$?" "0" diff --git a/src/program/lwaftr/tests/config/test-config-listen.sh b/src/program/lwaftr/tests/config/test-config-listen.sh index 29de3c9d60..3cb92c886a 100755 --- a/src/program/lwaftr/tests/config/test-config-listen.sh +++ b/src/program/lwaftr/tests/config/test-config-listen.sh @@ -1,63 +1,59 @@ #!/usr/bin/env bash -## This checks it can listen, send a command and get a -## response. It only tests the socket method of communicating -## with the listen command due to the difficulties of testing -## interactive scripts. -SKIPPED_CODE=43 - -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" 1>&2 - exit 1 -fi - -# Verify we have the "nc" tool, used to communicate with sockets -# if we don't have it we just have to skip this test. -which nc &> /dev/null -if [[ $? -ne 0 ]]; then - echo "No 'nc' tool present, unable to run test." 1&>2 - exit $SKIPPED_CODE -fi - -# Load the tools to be able to test stuff. -BASEDIR="`pwd`" -cd "`dirname \"$0\"`" -source tools.sh -cd $BASEDIR - -# Come up with a name for the lwaftr +## This checks it can listen, send a command and get a response. +## It only tests the socket method of communicating with the listen +## command due to the difficulties of testing interactive scripts. + +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh + +check_for_root + +# Verify we have the "nc" tool, used to communicate with sockets, +# and the "python" interpreter available. +# If we don't have them, we just have to skip this test. +check_command_available nc +check_command_available python + +# CONFIG_TEST_DIR is also set by the caller. +source ${CONFIG_TEST_DIR}/test_env.sh + +echo "Testing config listen" + +# Come up with a name for the lwaftr. SNABB_NAME="`random_name`" # Start the bench command. start_lwaftr_bench $SNABB_NAME -# Start the listen command with a socket +# Start the listen command with a socket. SOCKET_PATH="/tmp/snabb-test-listen-sock-$SNABB_NAME" ./snabb config listen --socket "$SOCKET_PATH" "$SNABB_NAME" &> /dev/null & -# It shouldn't take long but wait a short while for the socket -# to be created. +# Wait a short while for the socket to be created; it shouldn't take long. sleep 1 -# Create input and output fifo's to communicate. +# Create input and output FIFOs to communicate. LISTEN_IN=$(mktemp -u) LISTEN_OUT=$(mktemp -u) mkfifo "$LISTEN_IN" mkfifo "$LISTEN_OUT" -# Start a communication with the listen program +# Start a communication with the listen program. (cat "$LISTEN_IN" | nc -U "$SOCKET_PATH" > "$LISTEN_OUT") & -# Get the PID of nc so it can be easily stopped later +# Get the PID of nc so it can be easily stopped later. NC_PID=$! -# Finally, lets send a get command. +# Send a get command. GET_CMD="{ \"id\": \"0\", \"verb\": \"get\", \"path\": \"/routes/route[addr=1.2.3.4]/port\" }" echo "$GET_CMD" > "$LISTEN_IN" -# Sleep a short amount of time to let it respond, one second should be more than plenty +# Sleep a short amount of time to let it respond; +# one second should be more than plenty. sleep 1 -# Read the response from the listen command +# Read the response from the listen command. GET_CMD_RESPONSE=$(cat "$LISTEN_OUT") # Check the response as best I can, I'll use python as it's common to most. @@ -67,7 +63,7 @@ import json, sys print(json.loads(sys.stdin.read(200))[\"status\"])" ) -# Test the status is "ok" +# Test the status is "ok". assert_equal "$PARSED_GET_RESPONSE" "ok" # Finally end all the programs we've spawned. diff --git a/src/program/lwaftr/tests/config/test-config-remove.sh b/src/program/lwaftr/tests/config/test-config-remove.sh index 657a4cf0b1..dfe2bcb845 100755 --- a/src/program/lwaftr/tests/config/test-config-remove.sh +++ b/src/program/lwaftr/tests/config/test-config-remove.sh @@ -2,32 +2,32 @@ ## This adds a softwire section and then checks it can be got ## back and that all the values are as they should be. -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" 1>&2 - exit 1 -fi - -# Load the tools to be able to test stuff. -BASEDIR="`pwd`" -cd "`dirname \"$0\"`" -source tools.sh -cd $BASEDIR - -# Come up with a name for the lwaftr +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh + +check_for_root + +# CONFIG_TEST_DIR is also set by the caller. +source ${CONFIG_TEST_DIR}/test_env.sh + +echo "Testing config remove" + +# Come up with a name for the lwaftr. SNABB_NAME="`random_name`" # Start the bench command. start_lwaftr_bench $SNABB_NAME -# Firstly lets verify that the thing we want to remove actually exists +# Verify that the thing we want to remove actually exists. ./snabb config get "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null assert_equal "$?" "0" -# Then lets remove it +# Remove it. ./snabb config remove "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null assert_equal "$?" "0" -# Then lets verify we can't find it +# Verify we can't find it. ./snabb config get "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null || true assert_equal "$?" "0" diff --git a/src/program/lwaftr/tests/config/test-config-set.sh b/src/program/lwaftr/tests/config/test-config-set.sh index 818133179e..719730a029 100755 --- a/src/program/lwaftr/tests/config/test-config-set.sh +++ b/src/program/lwaftr/tests/config/test-config-set.sh @@ -2,18 +2,18 @@ ## This checks you can set values, it'll then perform a get to ## verify the value set is the value that is got too. -if [[ $EUID -ne 0 ]]; then - echo "This script must be run as root" 1>&2 - exit 1 -fi - -# Load the tools to be able to test stuff. -BASEDIR="`pwd`" -cd "`dirname \"$0\"`" -source tools.sh -cd $BASEDIR - -# Come up with a name for the lwaftr +# TEST_DIR is set by the caller, and passed onward. +export TEST_DIR +source ${TEST_DIR}/common.sh + +check_for_root + +# CONFIG_TEST_DIR is also set by the caller. +source ${CONFIG_TEST_DIR}/test_env.sh + +echo "Testing config set" + +# Come up with a name for the lwaftr. SNABB_NAME="`random_name`" # Start the bench command. @@ -33,14 +33,14 @@ TEST_PSID="0" SET_IP="`./snabb config get \"$SNABB_NAME\" \"/softwire-config/binding-table/softwire[ipv4=$TEST_IPV4][psid=$TEST_PSID]/b4-ipv6\"`" assert_equal "$SET_IP" "$TEST_IPV6" -# Check that the value above I just set is the same in the IETF schema -# We actually need to look this up backwards, lets just check the same -# IPv4 address was used as was used to set it above. +# Check that the value we just set is the same in the IETF schema. +# We actually need to look this up backwards, let's just check the same +# IPv4 address as was used to set it above. IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=::1]/binding-ipv4-addr" IPV4_ADDR="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" assert_equal "$IPV4_ADDR" "$TEST_IPV4" -# Also check the portset, the IPv4 address alone isn't unique +# Also check the portset, the IPv4 address alone isn't unique. IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=::1]/port-set/psid" PSID="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" assert_equal "$PSID" "$TEST_PSID" diff --git a/src/program/lwaftr/tests/config/test_env.sh b/src/program/lwaftr/tests/config/test_env.sh new file mode 100755 index 0000000000..69e6fa3a45 --- /dev/null +++ b/src/program/lwaftr/tests/config/test_env.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +function random_name { + cat /dev/urandom | tr -dc 'a-z' | fold -w 20 | head -n 1 +} + +# TEST_DIR is set by the caller. +DATA_DIR="${TEST_DIR}/data" +BENCHDATA_DIR="${TEST_DIR}/benchdata" + +# Start the lwaftr process. The process should end when the script ends; +# however, if something goes wrong and it doesn't end correctly, +# a duration is set to prevent it running indefinitely. +function start_lwaftr_bench { + ./snabb lwaftr bench --reconfigurable --bench-file /dev/null --name "$1" \ + --duration 30 \ + program/lwaftr/tests/data/icmp_on_fail.conf \ + program/lwaftr/tests/benchdata/ipv{4,6}-0550.pcap &> /dev/null & + + # Not ideal, but it takes a little time for the lwaftr to properly start. + sleep 2 +} + +function stop_lwaftr_bench { + # Get the job number for lwaftr bench. + local jobid="`jobs | grep -i \"lwaftr bench\" | awk '{print $1}' | tr -d '[]+'`" + kill -15 "%$jobid" + # Wait until it's shutdown. + wait &> /dev/null +} + +function stop_if_running { + # Check if it's running, if not, job done. + kill -0 "$1" &> /dev/null + if [[ $? -ne 0 ]]; then + return + fi + + # It's running, try and close it nicely. + kill -15 "$1" + wait &> /dev/null +} diff --git a/src/program/lwaftr/tests/config/tools.sh b/src/program/lwaftr/tests/config/tools.sh deleted file mode 100755 index 978baedde4..0000000000 --- a/src/program/lwaftr/tests/config/tools.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -function produce_error { - (>&2 echo $1) - exit 1 -} - -function random_name { - cat /dev/urandom | tr -dc 'a-z' | fold -w 20 | head -n 1 -} - - -# Takes two paremters and checks their equality. -# It takes an optional third argument which will -# be displayed if it fails the equality check. -# e.g. -# $ assert_equal "yellow "cat" -> error -# $ assert_equal "banana" "banana" -> nothing (valid) -function assert_equal { - if [[ -z "$2" ]]; then - produce_error "assert_equals: Not enough arguments." - exit 1 - fi - if [[ "$1" == "$2" ]]; then - return - else - if [[ "$3" == "" ]]; then - produce_error "Assert error: $1 != $2" - else - produce_error "Assert error: $3" - fi - fi -} - -# This starts the lwaftr process. The process should end when the script -# ends however, if something goes wrong and it doesn't end correctly, a -# duration is set to prevent it running indefinitely. -function start_lwaftr_bench { - ./snabb lwaftr bench --reconfigurable --bench-file /dev/null --name "$1" \ - --duration 30 \ - program/lwaftr/tests/data/icmp_on_fail.conf \ - program/lwaftr/tests/benchdata/ipv{4,6}-0550.pcap &> /dev/null & - - # This isn't ideal but it takes a little time for the lwaftr to properly start - sleep 2 -} - -function stop_lwaftr_bench { - # Get the job number for lwaftr bench - local jobid="`jobs | grep -i \"lwaftr bench\" | awk '{print $1}' | tr -d '[]+'`" - kill -15 "%$jobid" - # Wait until it's shutdown. - wait &> /dev/null -} - -function stop_if_running { - # Check if it's running, if not, job done. - kill -0 "$1" &> /dev/null - if [[ "$?" -ne 0 ]]; then - return - fi - - # It's running, lets try and close it nicely - kill -15 "$1" - wait &> /dev/null -} From 343bd31987bca0f7d99e816219bef2c7759b2fe0 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 13 Feb 2017 16:28:19 +0100 Subject: [PATCH 549/631] #737 - Add name leaf to snabb-softwire-v1 --- src/lib/yang/snabb-softwire-v1.yang | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 9669e74f7a..0ee4fc6b30 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -126,7 +126,10 @@ module snabb-softwire-v1 { } } - + leaf name { + type string; + description "Name of lwAFTR instance."; + } container external-interface { description From fae70bab5ac0d33105097bae8448aae793127122 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 13 Feb 2017 16:28:32 +0100 Subject: [PATCH 550/631] #737 - Support name by --name or snabb-softwire-v1 This adds the ability to define the lwAFTR's name in the YANG lwAFTR config or the current method of the `--name` parameter on the command line. The order of presidence is: 1. Use command line argument first (i.e. --name mylwaftr) 2. Use the config parameter at softwire-config.name 3. Don't set a name if no name has been provided. NB: This will fail if you try to use the same name twice, they must be unique. This may happen if a name is defined in a config file and it's not overriden for subsequent instances. --- src/program/lwaftr/bench/bench.lua | 10 +++++++++- src/program/lwaftr/run/run.lua | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index acc3c4bb9b..6cd98d068d 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -44,7 +44,15 @@ function run(args) local opts, scheduling, conf_file, inv4_pcap, inv6_pcap = parse_args(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) - if opts.name then engine.claim_name(opts.name) end + -- If there is a name defined on the command line, it should override + -- anything defined in the config. + if opts.name then + conf.softwire_config.name = opts.name + end + + if conf.softwire_config.name ~= nil then + engine.claim_name(opts.name) + end local graph = config.new() if opts.reconfigurable then diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 01e56558ad..0d80435a16 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -137,7 +137,15 @@ function run(args) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local use_splitter = requires_splitter(opts, conf) - if opts.name then engine.claim_name(opts.name) end + -- If there is a name defined on the command line, it should override + -- anything defined in the config. + if opts.name then + conf.softwire_config.name = opts.name + end + + if conf.softwire_config.name ~= nil then + engine.claim_name(opts.name) + end local c = config.new() local setup_fn, setup_args From 7eb419ec55cf3bd2dc7bf2ab363e7d413b3b5c1a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 13 Feb 2017 16:28:36 +0100 Subject: [PATCH 551/631] #737 - Add rename program support and fixes bug in claim_name This adds a function to provide provide a current lwaftr's name and have it change that name. This involves changing the symlinks on the file system that provide a name and link to the instance. The change also changes the code so that it doesn't rely on it being called and managed in the same process. --- src/core/app.lua | 64 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index fa96a06278..32f5327e93 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -135,19 +135,26 @@ end -- The name given to the function must be unique; if a name has been used before -- by an active process the function will error displaying an appropriate error -- message. The program can only claim one name, successive calls will produce --- an error. -function claim_name(name) - configuration[name] = name +-- an error.If the `pid` parameter is provided it will claim the name for that +-- process rather than the current process. +-- +-- NB: To change the name of a program please see rename_program +function claim_name(name, pid) + local procpid = pid or S.getpid() local bynamedir = "by-name/" local namedir_fq = named_program_root .. "/" .. name - local piddir = shm.root .. "/" .. S.getpid() + local piddir = shm.root .. "/" .. procpid local backlinkdir = piddir.."/name" + -- Lookup if a name exists already - we must do this as it can be changed + -- elsewhere, including other processes. + local currentname = S.readlink(backlinkdir) + -- Verify that the by-name directory exists. shm.mkdir(bynamedir) -- Verify that we've not already claimed a name - assert(configuration.name == nil, "Name already claimed, cannot claim: "..name) + assert(currentname == nil, "Name already claimed, cannot claim: "..name) -- Create the new symlink. assert(S.symlink(piddir, namedir_fq)) @@ -156,6 +163,35 @@ function claim_name(name) assert(S.symlink(namedir_fq, backlinkdir)) end +-- Renames a program to a new name +-- +-- This renames an existing named app with a new name. If the app has not +-- claimed a name, it will display an error. The name must be unique with +-- other snabb programs running on the system. +-- +-- NB: To claim a name for the first time please see claim_name +function rename_program(old, new) + local old_fq = named_program_root .. "/" .. old + local new_fq = named_program_root .. "/" .. new + local backlink = shm.root .. "/" .. S.getpid() .. "/name" + + -- Get the PID for the old name. + local piddir = assert(S.readlink(old_fq), "No program with name '"..old.."'") + + -- Now secure the new name + assert(S.symlink(piddir, new_fq)) + + -- Remove the previous symlink + assert(S.unlink(old_fq)) + + -- Replace the backlink. + assert(S.unlink(backlink)) + assert(S.symlink(new_fq, backlink)) + + -- Change the name in configuration + configuration.name = new +end + -- Enumerates the named programs with their PID -- -- This returns a table programs with the key being the name of the program @@ -652,16 +688,24 @@ function selftest () -- Test claiming and enumerating app names local basename = "testapp" - claim_name(basename.."1") + local progname = basename.."1" + claim_name(progname) -- Ensure to claim two names fails assert(not pcall(claim_name, basename.."2")) -- Check if it can be enumerated. - local progs = enumerate_named_programs() - assert(progs) - assert(progs["testapp1"]) + local progs = assert(enumerate_named_programs()) + assert(progs[progname]) -- Ensure that trying to take the same name fails - assert(not pcall(claim_name, basename.."1")) + assert(not pcall(claim_name, progname)) + + -- Ensure changing the name succeeds + local newname = basename.."2" + rename_program(progname, newname) + local progs = assert(enumerate_named_programs()) + assert(progs[progname] == nil) + assert(progs[newname]) + end From 60af0b150e0b851586ec6173172fb0648712905d Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 13 Feb 2017 16:28:39 +0100 Subject: [PATCH 552/631] #737 - Add support for changing name via snabb set This adds support for the name to be set (for the first time) or altered by using the `snabb config set` command on the `/softwire-config/name` leaf. This will claim the new name and if there was one release back (free) the old name so it could be used by another snabb process. An example of a name change command could be: $ snabb config set oldname "/softwire-config/name" "newname" --- src/apps/config/leader.lua | 8 ++-- src/apps/config/support/snabb-softwire-v1.lua | 42 ++++++++++++++++++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 807cdbc403..9165caedba 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -169,8 +169,9 @@ local function path_setter_for_grammar(grammar, path) assert(grammar.type == 'struct') local tail_id = data.normalize_id(tail_name) return function(config, subconfig) - getter(config)[tail_id] = subconfig - return config + local newconfig = lib.deepcopy(config) + getter(newconfig)[tail_id] = subconfig + return newconfig end end @@ -409,7 +410,8 @@ function Leader:update_configuration (update_fn, verb, path, ...) local new_config = update_fn(self.current_configuration, ...) local new_app_graph = self.setup_fn(new_config) local actions = self.support.compute_config_actions( - self.current_app_graph, new_app_graph, to_restart, verb, path, ...) + self.current_app_graph, new_app_graph, to_restart, verb, path, + self.followers, ...) self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph self.current_configuration = new_config diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 59f072ed92..a7f00cc07d 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -1,6 +1,9 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) +local S = require("syscall") local ffi = require('ffi') +local shm = require('core.shm') +local lib = require('core.lib') local app = require('core.app') local equal = require('core.lib').equal local dirname = require('core.lib').dirname @@ -38,6 +41,34 @@ local function get_softwire_grammar() return softwire_grammar end +local function change_lwaftr_name_actions(old_graph, new_graph, followers) + -- We should perform the name change (rename) of the lwaftr instance here + -- as it involves making syscalls and isn't paticually performant. It + -- shouldn't matter if it's done in another process from the lwaftr as it's + -- mainly used externally. + assert(old_graph.apps.lwaftr) + assert(new_graph.apps.lwaftr) + + local function lwaftr_pid_by_follower_pid(follower_pid) + assert(follower_pid) + local lwaftrdir = assert(S.readlink(shm.root .. "/" .. follower_pid .. "/group")) + return tonumber(lib.basename(lwaftrdir)) + end + + local oldname = old_graph.apps.lwaftr.arg.softwire_config.name + local newname = new_graph.apps.lwaftr.arg.softwire_config.name + + if oldname == nil then + for _, follower in pairs(followers) do + app.claim_name(newname, lwaftr_pid_by_follower_pid(follower.pid)) + end + else + app.rename_program(oldname, newname) + end + + return {} +end + local function remove_softwire_entry_actions(app_graph, path) assert(app_graph.apps['lwaftr']) path = path_mod.parse_path(path) @@ -49,12 +80,14 @@ local function remove_softwire_entry_actions(app_graph, path) end local function compute_config_actions(old_graph, new_graph, to_restart, - verb, path, arg) + verb, path, followers, arg) if verb == 'add' and path == '/softwire-config/binding-table/softwire' then return add_softwire_entry_actions(new_graph, arg) elseif (verb == 'remove' and path:match('^/softwire%-config/binding%-table/softwire')) then return remove_softwire_entry_actions(new_graph, path) + elseif (verb == 'set' and path == '/softwire-config/name') then + return change_lwaftr_name_actions(old_graph, new_graph, followers) else return generic.compute_config_actions( old_graph, new_graph, to_restart, verb, path, arg) @@ -81,6 +114,8 @@ local function compute_apps_to_restart_after_configuration_update( elseif (verb == 'remove' and path:match('^/softwire%-config/binding%-table/softwire')) then return {} + elseif (verb == 'set' and path == '/softwire-config/name') then + return {} else return generic.compute_apps_to_restart_after_configuration_update( schema_name, configuration, verb, path, in_place_dependencies) @@ -547,6 +582,11 @@ local function ietf_softwire_translator () instance.binding_table.binding_entry[k] = v end end + elseif (verb == 'set' and path == "/softwire-config/name") then + local br = cached_config.softwire_config.binding.br + for _, instance in cltable.pairs(br.br_instances.br_instance) do + instance.name = data + end else cached_config = nil end From 0467caaae1d11e96b37445d0bc9175669197d547 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 13 Feb 2017 16:28:42 +0100 Subject: [PATCH 553/631] #737 - Add documentation for name leaf (inc. presidence) --- src/lib/yang/snabb-softwire-v1.yang | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/lib/yang/snabb-softwire-v1.yang b/src/lib/yang/snabb-softwire-v1.yang index 0ee4fc6b30..54fec51aa1 100644 --- a/src/lib/yang/snabb-softwire-v1.yang +++ b/src/lib/yang/snabb-softwire-v1.yang @@ -128,7 +128,19 @@ module snabb-softwire-v1 { leaf name { type string; - description "Name of lwAFTR instance."; + description + "Name of lwAFTR instance. This must be unique amongst the Snabb + processes on the system. This may be specified either here, in the + YANG configuration or via the command line when the lwAFTR is started. + + The order of presidence for this leaf is as followers: + 1. The name set on an already running lwAFTR instance via snabb set. + 2. A command line option to specify the name upon starting the lwAFTR + instance (i.e. overriding this value). + 3. The value here in the configuration when starting a lwaftr instance. + + If no name is specified the lwaftr can be referred to using the PID of + the lwAFTR process on the system."; } container external-interface { From 5355f1be99f1340f8626afc1042340e76d017e57 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 13 Feb 2017 16:28:44 +0100 Subject: [PATCH 554/631] #737 - Add support for name leaf in ietf-softwire translation --- src/apps/config/support/snabb-softwire-v1.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index a7f00cc07d..ad680e1581 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -254,6 +254,7 @@ local function ietf_softwire_translator () local br_instance, br_instance_key_t = cltable_for_grammar(get_ietf_br_instance_grammar()) br_instance[br_instance_key_t({id=1})] = { + name = native_config.softwire_config.name, tunnel_payload_mtu = native_config.softwire_config.internal_interface.mtu, tunnel_path_mru = native_config.softwire_config.external_interface.mtu, -- FIXME: There's no equivalent of softwire-num-threshold in snabb-softwire-v1. @@ -326,6 +327,11 @@ local function ietf_softwire_translator () if path[#path].name == 'softwire-num-threshold' then error('not yet implemented: softwire-num-threshold') end + if path[#path].name == 'name' then + return {{'set', {schema='snabb-softwire-v1', + path="/softwire-config/name", + config=arg}}} + end error('unrecognized leaf: '..path[#path].name) end From fd9d038ab2d6b432170946b6715fdf5ba2374a99 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 13 Feb 2017 16:40:07 +0100 Subject: [PATCH 555/631] Add a selftest for the lwaftr bench command (#739) --- src/program/lwaftr/bench/selftest.sh | 24 ++++++++++++++++++++++++ src/program/lwaftr/selftest.sh | 18 +----------------- 2 files changed, 25 insertions(+), 17 deletions(-) create mode 100755 src/program/lwaftr/bench/selftest.sh diff --git a/src/program/lwaftr/bench/selftest.sh b/src/program/lwaftr/bench/selftest.sh new file mode 100755 index 0000000000..c728f005c6 --- /dev/null +++ b/src/program/lwaftr/bench/selftest.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +TEST_DIR="./program/lwaftr/tests" + +source ${TEST_DIR}/common.sh + +check_for_root + +echo "Testing lwaftr bench" + +DATA_DIR="${TEST_DIR}/data" +BENCHDATA_DIR="${TEST_DIR}/benchdata" + +./snabb lwaftr bench --duration 1 --bench-file bench.csv \ + ${DATA_DIR}/icmp_on_fail.conf \ + ${BENCHDATA_DIR}/ipv{4,6}-0550.pcap &> /dev/null +assert_equal $? 0 "lwaftr bench failed with error code $?" +assert_file_exists ./bench.csv --remove + +./snabb lwaftr bench --reconfigurable --duration 1 --bench-file bench.csv \ + ${DATA_DIR}/icmp_on_fail.conf \ + ${BENCHDATA_DIR}/ipv{4,6}-0550.pcap &> /dev/null +assert_equal $? 0 "lwaftr bench --reconfigurable failed with error code $?" +assert_file_exists ./bench.csv --remove diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh index 108384f542..8d51c456b3 100755 --- a/src/program/lwaftr/selftest.sh +++ b/src/program/lwaftr/selftest.sh @@ -10,23 +10,7 @@ if [[ $EUID != 0 ]]; then exit 1 fi -# These are tests for lwaftr front ends. - -echo "Testing snabb lwaftr bench" -./snabb lwaftr bench -D 0.1 ${TDIR}/icmp_on_fail.conf \ - ${TDIR}/tcp-frominet-bound.pcap ${TDIR}/tcp-fromb4-ipv6.pcap - -echo "Testing snabb lwaftr bench --reconfigurable" -./snabb lwaftr bench --reconfigurable -D 1 ${TDIR}/icmp_on_fail.conf \ - ${TDIR}/tcp-frominet-bound.pcap ${TDIR}/tcp-fromb4-ipv6.pcap & - -LEADER_PID=$! - -sleep 0.1 -./snabb config get $LEADER_PID / - - -# The rest of the tests require real hardware +# The tests require real hardware if [ -z "$SNABB_PCI0" ]; then echo "Skipping tests which require real hardware, SNABB_PCI0 not set" From 53d3d7a29fcda51261e775d1ec5667298d9bdc52 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Tue, 14 Feb 2017 11:01:56 +0100 Subject: [PATCH 556/631] Bench command: turn off leader busywait, mention --reconfigurable in README (#738) --- src/program/lwaftr/bench/README | 11 +++++++---- src/program/lwaftr/bench/bench.lua | 5 ++++- src/program/lwaftr/run/README | 2 ++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/program/lwaftr/bench/README b/src/program/lwaftr/bench/README index 652d9e0219..90501ffe2c 100644 --- a/src/program/lwaftr/bench/README +++ b/src/program/lwaftr/bench/README @@ -12,16 +12,19 @@ Usage: bench CONF IPV4-IN.PCAP IPV6-IN.PCAP Time (s),Decap. MPPS,Decap. Gbps,Encap. MPPS,Encap. Gbps -b FILENAME, --bench-file FILENAME - The file or path name to which benchmark data is - written. A simple filename or relative pathname - will be based on the current directory. Default - is "bench.csv". + The file or path name to which benchmark data is + written. A simple filename or relative pathname + will be based on the current directory. Default + is "bench.csv". -D DURATION, --duration DURATION Duration in seconds. -n NAME, --name NAME Sets the name for this program, which will be used as the identifier. This must be unique amongst other snabb processes. --cpu Bind lwAFTR bench to the given CPU + --reconfigurable Allow run-time configuration query and update. + This runs as two processes, a leader and a + follower, rather than just one. Run the lwAFTR with input from IPV4-IN.PCAP and IPV6-IN.PCAP. The bench command is used to get an idea of the raw speed of the lwaftr without diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index acc3c4bb9b..035ed74de6 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -80,6 +80,9 @@ function run(args) start_sampling_for_pid(S.getpid()) end - app.busywait = true + if not opts.reconfigurable then + -- The leader does not need all the CPU, only the followers do. + app.busywait = true + end app.main({duration=opts.duration}) end diff --git a/src/program/lwaftr/run/README b/src/program/lwaftr/run/README index 24ed6c9f08..e713c780a0 100644 --- a/src/program/lwaftr/run/README +++ b/src/program/lwaftr/run/README @@ -10,6 +10,8 @@ Required arguments: Optional arguments: --reconfigurable Allow run-time configuration query and update. + This runs as two processes, a leader and a + follower, rather than just one. --virtio Use virtio-net interfaces instead of Intel 82599 --ring-buffer-size Set Intel 82599 receive buffer size --cpu Bind the lwAFTR to the given CPU From 2e2b3d7fa69e4447f7844bd256eaa9041b5038f0 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 14 Feb 2017 14:28:42 +0100 Subject: [PATCH 557/631] #737 - Move name claiming to setup_fn and add unclaim_name This changes where name claiming occurs. The `claim_name` call now occurs in the `setup_fn` of the lwaftr. This is also how the leader changes the name of a running lwaftr instance. The `claim_name` has been reachitected to deal with name changes as well as successive calls with the same name to allow it to be used how it is now in `setup_fn`. This also adds the `unclaim_name` allowing for release or "free"ing of a claimed name. Successive calls to `claim_name` will now claim the new name and if successful it will then release/unclaim the previous name. --- src/apps/config/leader.lua | 8 +- src/apps/config/support/snabb-softwire-v1.lua | 35 +----- src/core/app.lua | 101 +++++++++--------- src/program/lwaftr/bench/bench.lua | 4 - src/program/lwaftr/run/run.lua | 4 - src/program/lwaftr/setup.lua | 9 ++ 6 files changed, 66 insertions(+), 95 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 9165caedba..807cdbc403 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -169,9 +169,8 @@ local function path_setter_for_grammar(grammar, path) assert(grammar.type == 'struct') local tail_id = data.normalize_id(tail_name) return function(config, subconfig) - local newconfig = lib.deepcopy(config) - getter(newconfig)[tail_id] = subconfig - return newconfig + getter(config)[tail_id] = subconfig + return config end end @@ -410,8 +409,7 @@ function Leader:update_configuration (update_fn, verb, path, ...) local new_config = update_fn(self.current_configuration, ...) local new_app_graph = self.setup_fn(new_config) local actions = self.support.compute_config_actions( - self.current_app_graph, new_app_graph, to_restart, verb, path, - self.followers, ...) + self.current_app_graph, new_app_graph, to_restart, verb, path, ...) self:enqueue_config_actions(actions) self.current_app_graph = new_app_graph self.current_configuration = new_config diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index ad680e1581..923ad3cdde 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -1,9 +1,6 @@ -- Use of this source code is governed by the Apache 2.0 license; see COPYING. module(..., package.seeall) -local S = require("syscall") local ffi = require('ffi') -local shm = require('core.shm') -local lib = require('core.lib') local app = require('core.app') local equal = require('core.lib').equal local dirname = require('core.lib').dirname @@ -41,34 +38,6 @@ local function get_softwire_grammar() return softwire_grammar end -local function change_lwaftr_name_actions(old_graph, new_graph, followers) - -- We should perform the name change (rename) of the lwaftr instance here - -- as it involves making syscalls and isn't paticually performant. It - -- shouldn't matter if it's done in another process from the lwaftr as it's - -- mainly used externally. - assert(old_graph.apps.lwaftr) - assert(new_graph.apps.lwaftr) - - local function lwaftr_pid_by_follower_pid(follower_pid) - assert(follower_pid) - local lwaftrdir = assert(S.readlink(shm.root .. "/" .. follower_pid .. "/group")) - return tonumber(lib.basename(lwaftrdir)) - end - - local oldname = old_graph.apps.lwaftr.arg.softwire_config.name - local newname = new_graph.apps.lwaftr.arg.softwire_config.name - - if oldname == nil then - for _, follower in pairs(followers) do - app.claim_name(newname, lwaftr_pid_by_follower_pid(follower.pid)) - end - else - app.rename_program(oldname, newname) - end - - return {} -end - local function remove_softwire_entry_actions(app_graph, path) assert(app_graph.apps['lwaftr']) path = path_mod.parse_path(path) @@ -80,14 +49,14 @@ local function remove_softwire_entry_actions(app_graph, path) end local function compute_config_actions(old_graph, new_graph, to_restart, - verb, path, followers, arg) + verb, path, arg) if verb == 'add' and path == '/softwire-config/binding-table/softwire' then return add_softwire_entry_actions(new_graph, arg) elseif (verb == 'remove' and path:match('^/softwire%-config/binding%-table/softwire')) then return remove_softwire_entry_actions(new_graph, path) elseif (verb == 'set' and path == '/softwire-config/name') then - return change_lwaftr_name_actions(old_graph, new_graph, followers) + return {} else return generic.compute_config_actions( old_graph, new_graph, to_restart, verb, path, arg) diff --git a/src/core/app.lua b/src/core/app.lua index 32f5327e93..e6494d1fcc 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -130,68 +130,68 @@ function configure (new_config) counter.add(configs) end +-- Removes the claim on a name, freeing it for other programs. +-- +-- This relinquish a claim on a name if one exists. if the name does not +-- exist it will raise an error with an error message. +function unclaim_name(claimed_name) + local name = assert(claimed_name or configuration.name, "No claim to name.") + local name_fq = named_program_root .. "/" .. name + local piddir = assert(S.readlink(name_fq)) + local backlink = piddir .. "/name" + + -- First unlink the backlink + assert(S.unlink(backlink)) + + -- Remove the actual namedir + assert(S.unlink(name_fq)) + + -- Remove from the name from the configuration + configuration.name = nil +end + -- Claims a name for a program so it's identified by name by other processes. -- -- The name given to the function must be unique; if a name has been used before -- by an active process the function will error displaying an appropriate error --- message. The program can only claim one name, successive calls will produce --- an error.If the `pid` parameter is provided it will claim the name for that --- process rather than the current process. --- --- NB: To change the name of a program please see rename_program -function claim_name(name, pid) - local procpid = pid or S.getpid() - local bynamedir = "by-name/" +-- message.Successive calls to claim_name with the same name will return with +-- inaction. If the program has already claimed a name and this is called with +-- a different name, it will attempt to claim the new name and then unclaim the +-- old name. If an problem occurs whilst claiming the new name, the old name +-- will remain claimed. +function claim_name(name) local namedir_fq = named_program_root .. "/" .. name + local procpid = S.getpid() local piddir = shm.root .. "/" .. procpid local backlinkdir = piddir.."/name" - -- Lookup if a name exists already - we must do this as it can be changed - -- elsewhere, including other processes. - local currentname = S.readlink(backlinkdir) + + -- If we're being asked to claim the name we already have, return false. + if configuration.name == name then + return + end + + -- If we have an name already, we need to keep it around to unclaim it later. + if configuration.name ~= nil then + oldname = configuration.name + end -- Verify that the by-name directory exists. - shm.mkdir(bynamedir) + shm.mkdir("by-name/") + + -- Create the new symlink (name has probably been taken if this fails). + assert(S.symlink(piddir, namedir_fq), "Name already taken.") - -- Verify that we've not already claimed a name - assert(currentname == nil, "Name already claimed, cannot claim: "..name) + -- We've successfully secured the new name, so we can unclaim the old now. + if configuration.name ~= nil then unclaim_name(configuration.name) end - -- Create the new symlink. - assert(S.symlink(piddir, namedir_fq)) + -- Save our current name so we know what it is later. + configuration.name = name -- Create a backlink so to the symlink so we can easily cleanup assert(S.symlink(namedir_fq, backlinkdir)) end --- Renames a program to a new name --- --- This renames an existing named app with a new name. If the app has not --- claimed a name, it will display an error. The name must be unique with --- other snabb programs running on the system. --- --- NB: To claim a name for the first time please see claim_name -function rename_program(old, new) - local old_fq = named_program_root .. "/" .. old - local new_fq = named_program_root .. "/" .. new - local backlink = shm.root .. "/" .. S.getpid() .. "/name" - - -- Get the PID for the old name. - local piddir = assert(S.readlink(old_fq), "No program with name '"..old.."'") - - -- Now secure the new name - assert(S.symlink(piddir, new_fq)) - - -- Remove the previous symlink - assert(S.unlink(old_fq)) - - -- Replace the backlink. - assert(S.unlink(backlink)) - assert(S.symlink(new_fq, backlink)) - - -- Change the name in configuration - configuration.name = new -end - -- Enumerates the named programs with their PID -- -- This returns a table programs with the key being the name of the program @@ -691,9 +691,6 @@ function selftest () local progname = basename.."1" claim_name(progname) - -- Ensure to claim two names fails - assert(not pcall(claim_name, basename.."2")) - -- Check if it can be enumerated. local progs = assert(enumerate_named_programs()) assert(progs[progname]) @@ -703,9 +700,15 @@ function selftest () -- Ensure changing the name succeeds local newname = basename.."2" - rename_program(progname, newname) + claim_name(newname) local progs = assert(enumerate_named_programs()) assert(progs[progname] == nil) assert(progs[newname]) + + -- Ensure unclaiming the name occurs + unclaim_name() + local progs = enumerate_named_programs() + assert(progs[newname] == nil) + assert(configuration.name == nil) end diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 6cd98d068d..5c0a831db6 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -50,10 +50,6 @@ function run(args) conf.softwire_config.name = opts.name end - if conf.softwire_config.name ~= nil then - engine.claim_name(opts.name) - end - local graph = config.new() if opts.reconfigurable then setup.reconfigurable(scheduling, setup.load_bench, graph, conf, diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 0d80435a16..bf266a0d9e 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -143,10 +143,6 @@ function run(args) conf.softwire_config.name = opts.name end - if conf.softwire_config.name ~= nil then - engine.claim_name(opts.name) - end - local c = config.new() local setup_fn, setup_args if opts.virtio_net then diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 8fce523fd3..fa312af922 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -527,6 +527,15 @@ function reconfigurable(scheduling, f, graph, conf, ...) local function setup_fn(conf) local graph = config.new() f(graph, conf, unpack(args)) + local name = graph.apps.lwaftr.arg.softwire_config.name + if name then + local succ, err = pcall(engine.claim_name, name) + if succ == false then + local oldname = engine.configuration.name + graph.apps.lwaftr.arg.softwire_config.name = oldname + end + assert(succ, err) + end return graph end From 99aff07fd259a965f40f0ead5520ba65a4f87373 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 14 Feb 2017 14:42:15 +0100 Subject: [PATCH 558/631] #737 - Remove test that successive calls to claim_name fails --- src/core/app.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/core/app.lua b/src/core/app.lua index e6494d1fcc..e9fe160f7e 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -695,9 +695,6 @@ function selftest () local progs = assert(enumerate_named_programs()) assert(progs[progname]) - -- Ensure that trying to take the same name fails - assert(not pcall(claim_name, progname)) - -- Ensure changing the name succeeds local newname = basename.."2" claim_name(newname) From f19182ad8f72e76a37859c1201d25de95eb8ec78 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 14 Feb 2017 17:46:17 +0100 Subject: [PATCH 559/631] Change configuration.name and improve name changes This changes the once `configuration.name` attribute to be it's own variable `program_name`. This is because `configuration` is an app graph and it seems wrong to pollute an app graph with the name. It also improves claiming the name within the lwAFTR. This is done by having a `switch_name` function which takes an app graph, tries to claim the name (if one exists) and if it fails, gracefully restore the old name back onto the app graph's configuration. It also cleans up some formatting. --- src/apps/config/support/snabb-softwire-v1.lua | 2 +- src/core/app.lua | 26 +++++++++---------- src/program/lwaftr/setup.lua | 26 ++++++++++++------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 923ad3cdde..cd86d5781a 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -223,7 +223,7 @@ local function ietf_softwire_translator () local br_instance, br_instance_key_t = cltable_for_grammar(get_ietf_br_instance_grammar()) br_instance[br_instance_key_t({id=1})] = { - name = native_config.softwire_config.name, + name = native_config.softwire_config.name, tunnel_payload_mtu = native_config.softwire_config.internal_interface.mtu, tunnel_path_mru = native_config.softwire_config.external_interface.mtu, -- FIXME: There's no equivalent of softwire-num-threshold in snabb-softwire-v1. diff --git a/src/core/app.lua b/src/core/app.lua index e9fe160f7e..4482067752 100644 --- a/src/core/app.lua +++ b/src/core/app.lua @@ -27,7 +27,10 @@ local use_restart = false test_skipped_code = 43 -- Set the directory for the named programs. -named_program_root = shm.root .. "/" .. "by-name" +local named_program_root = shm.root .. "/" .. "by-name" + +-- The currently claimed name (think false = nil but nil makes strict.lua unhappy). +program_name = false -- The set of all active apps and links in the system, indexed by name. app_table, link_table = {}, {} @@ -135,7 +138,7 @@ end -- This relinquish a claim on a name if one exists. if the name does not -- exist it will raise an error with an error message. function unclaim_name(claimed_name) - local name = assert(claimed_name or configuration.name, "No claim to name.") + local name = assert(claimed_name or program_name, "No claim to name.") local name_fq = named_program_root .. "/" .. name local piddir = assert(S.readlink(name_fq)) local backlink = piddir .. "/name" @@ -147,7 +150,7 @@ function unclaim_name(claimed_name) assert(S.unlink(name_fq)) -- Remove from the name from the configuration - configuration.name = nil + program_name = false end -- Claims a name for a program so it's identified by name by other processes. @@ -165,17 +168,11 @@ function claim_name(name) local piddir = shm.root .. "/" .. procpid local backlinkdir = piddir.."/name" - -- If we're being asked to claim the name we already have, return false. - if configuration.name == name then + if program_name == name then return end - -- If we have an name already, we need to keep it around to unclaim it later. - if configuration.name ~= nil then - oldname = configuration.name - end - -- Verify that the by-name directory exists. shm.mkdir("by-name/") @@ -183,10 +180,10 @@ function claim_name(name) assert(S.symlink(piddir, namedir_fq), "Name already taken.") -- We've successfully secured the new name, so we can unclaim the old now. - if configuration.name ~= nil then unclaim_name(configuration.name) end + if program_name ~= false then unclaim_name(program_name) end -- Save our current name so we know what it is later. - configuration.name = name + program_name = name -- Create a backlink so to the symlink so we can easily cleanup assert(S.symlink(namedir_fq, backlinkdir)) @@ -686,6 +683,9 @@ function selftest () main({duration = 4, report = {showapps = true}}) assert(app_table.app3 ~= orig_app3) -- should be restarted + -- Check one can't unclaim a name if no name is claimed. + assert(not pcall(unclaim_name)) + -- Test claiming and enumerating app names local basename = "testapp" local progname = basename.."1" @@ -706,6 +706,6 @@ function selftest () unclaim_name() local progs = enumerate_named_programs() assert(progs[newname] == nil) - assert(configuration.name == nil) + assert(not program_name) end diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index fa312af922..b038777b3c 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -524,18 +524,26 @@ end function reconfigurable(scheduling, f, graph, conf, ...) local args = {...} + local function switch_names(conf) + local currentname = engine.program_name + local name = conf.apps.lwaftr.arg.softwire_config.name + -- Don't do anything if the name isn't set. + if name == nil then + return + end + + local success, err = pcall(engine.claim_name, name) + if success == false then + -- Restore the previous name. + conf.apps.lwaftr.arg.softwire_config.name = currentname + assert(success, err) + end + end + local function setup_fn(conf) local graph = config.new() f(graph, conf, unpack(args)) - local name = graph.apps.lwaftr.arg.softwire_config.name - if name then - local succ, err = pcall(engine.claim_name, name) - if succ == false then - local oldname = engine.configuration.name - graph.apps.lwaftr.arg.softwire_config.name = oldname - end - assert(succ, err) - end + switch_names(graph) return graph end From 176773acbd82f1ec2fec704c540a55a1aeb4ede9 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Wed, 15 Feb 2017 17:09:50 +0100 Subject: [PATCH 560/631] Enhance NUMA selftest (#734) --- src/lib/numa.lua | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/src/lib/numa.lua b/src/lib/numa.lua index 0bedf65ab1..03534b8cd4 100644 --- a/src/lib/numa.lua +++ b/src/lib/numa.lua @@ -14,11 +14,13 @@ local pci = require("lib.hardware.pci") local bound_cpu local bound_numa_node +local node_path = '/sys/devices/system/node/node' +local MAX_CPU = 1023 + function cpu_get_numa_node (cpu) local node = 0 while true do - local node_dir = S.open('/sys/devices/system/node/node'..node, - 'rdonly, directory') + local node_dir = S.open(node_path..node, 'rdonly, directory') if not node_dir then return end local found = S.readlinkat(node_dir, 'cpu'..cpu) node_dir:close() @@ -28,7 +30,7 @@ function cpu_get_numa_node (cpu) end function has_numa () - local node1 = S.open('/sys/devices/system/node/node1', 'rdonly, directory') + local node1 = S.open(node_path..tostring(1), 'rdonly, directory') if not node1 then return false end node1:close() return true @@ -82,7 +84,7 @@ end function unbind_cpu () local cpu_set = S.sched_getaffinity() cpu_set:zero() - for i = 0, 1023 do cpu_set:set(i) end + for i = 0, MAX_CPU do cpu_set:set(i) end assert(S.sched_setaffinity(0, cpu_set)) bound_cpu = nil end @@ -126,18 +128,29 @@ function prevent_preemption(priority) end function selftest () + + function test_cpu(cpu) + local node = cpu_get_numa_node(cpu) + bind_to_cpu(cpu) + assert(bound_cpu == cpu) + assert(bound_numa_node == node) + assert(S.getcpu().cpu == cpu) + assert(S.getcpu().node == node) + bind_to_cpu(nil) + assert(bound_cpu == nil) + assert(bound_numa_node == node) + assert(S.getcpu().node == node) + bind_to_numa_node(nil) + assert(bound_cpu == nil) + assert(bound_numa_node == nil) + end + print('selftest: numa') - bind_to_cpu(0) - assert(bound_cpu == 0) - assert(bound_numa_node == 0) - assert(S.getcpu().cpu == 0) - assert(S.getcpu().node == 0) - bind_to_cpu(nil) - assert(bound_cpu == nil) - assert(bound_numa_node == 0) - assert(S.getcpu().node == 0) - bind_to_numa_node(nil) - assert(bound_cpu == nil) - assert(bound_numa_node == nil) + local cpu_set = S.sched_getaffinity() + for cpuid = 0, MAX_CPU do + if cpu_set:get(cpuid) then + test_cpu(cpuid) + end + end print('selftest: numa: ok') end From 56b924014884e1d6edf944c1fb33b1ffee8cf7fd Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Fri, 17 Feb 2017 17:57:58 +0100 Subject: [PATCH 561/631] Refactor config and query tests (#744) Improve error messages, disable tests for currently broken 'query --name' in non-reconfigurable mode --- src/program/lwaftr/query/selftest.sh | 4 +- src/program/lwaftr/query/tests/test_env.sh | 47 +++++++------------ src/program/lwaftr/query/tests/test_query.sh | 27 ++++++----- .../query/tests/test_query_reconfigurable.sh | 21 +++++---- src/program/lwaftr/tests/common.sh | 42 ++++++++++++----- src/program/lwaftr/tests/config/selftest.sh | 12 ++--- .../lwaftr/tests/config/test-config-add.sh | 12 ++--- .../tests/config/test-config-get-state.sh | 12 ++--- .../lwaftr/tests/config/test-config-get.sh | 19 ++++---- .../lwaftr/tests/config/test-config-listen.sh | 15 +++--- .../lwaftr/tests/config/test-config-remove.sh | 12 ++--- .../lwaftr/tests/config/test-config-set.sh | 13 +++-- src/program/lwaftr/tests/config/test_env.sh | 14 +++--- 13 files changed, 129 insertions(+), 121 deletions(-) diff --git a/src/program/lwaftr/query/selftest.sh b/src/program/lwaftr/query/selftest.sh index 825183f47a..a8b3266824 100755 --- a/src/program/lwaftr/query/selftest.sh +++ b/src/program/lwaftr/query/selftest.sh @@ -6,5 +6,5 @@ LWAFTR_DIR="./program/lwaftr" export TEST_DIR="${LWAFTR_DIR}/tests" export QUERY_TEST_DIR="${LWAFTR_DIR}/query/tests" -${QUERY_TEST_DIR}/test_query.sh -${QUERY_TEST_DIR}/test_query_reconfigurable.sh +${QUERY_TEST_DIR}/test_query.sh || exit $? +${QUERY_TEST_DIR}/test_query_reconfigurable.sh || exit $? diff --git a/src/program/lwaftr/query/tests/test_env.sh b/src/program/lwaftr/query/tests/test_env.sh index 6db4f66b1e..da083088e7 100644 --- a/src/program/lwaftr/query/tests/test_env.sh +++ b/src/program/lwaftr/query/tests/test_env.sh @@ -1,33 +1,18 @@ #!/usr/bin/env bash -TEMP_FILE=$(mktemp) +# TEST_DIR is set by the caller. +source ${TEST_DIR}/common.sh || exit $? -function tmux_launch { - command="$2 2>&1 | tee $3" - if [ -z "$tmux_session" ]; then - tmux_session=test_env-$$ - tmux new-session -d -n "$1" -s $tmux_session "$command" - else - tmux new-window -a -d -n "$1" -t $tmux_session "$command" - fi -} - -function kill_lwaftr { - ps aux | grep $SNABB_PCI0 | awk '{print $2}' | xargs kill 2>/dev/null -} +TEST_OUTPUT_FNAME=$(mktemp) -function cleanup { - kill_lwaftr - rm -f $TEMP_FILE +# Terminate the "lwaftr run" command, remove the output file, and exit. +function query_cleanup { + ps aux | grep $SNABB_PCI0 | grep -v "grep" | awk '{print $2}' | xargs kill 2> /dev/null + ps aux | grep "snabb lwaftr query" | grep -v "grep" | awk '{print $2}' | xargs kill 2> /dev/null + rm -f $TEST_OUTPUT_FNAME exit } -function fatal { - local msg=$1 - echo "Error: $msg" - exit 1 -} - function get_lwaftr_leader { local pids=$(ps aux | grep "\-\-reconfigurable" | grep $SNABB_PCI0 | grep -v "grep" | awk '{print $2}') for pid in ${pids[@]}; do @@ -61,21 +46,23 @@ function get_lwaftr_instance { } function test_lwaftr_query { - ./snabb lwaftr query $@ > $TEMP_FILE - local lineno=`cat $TEMP_FILE | wc -l` + ./snabb lwaftr query $@ > $TEST_OUTPUT_FNAME + local lineno=`cat $TEST_OUTPUT_FNAME | wc -l` if [[ $lineno -gt 1 ]]; then echo "Success: lwaftr query $*" else - fatal "lwaftr query $*" + cat $TEST_OUTPUT_FNAME + exit_on_error "Error: lwaftr query $*" fi } function test_lwaftr_query_no_counters { - ./snabb lwaftr query $@ > $TEMP_FILE - local lineno=`cat $TEMP_FILE | wc -l` + ./snabb lwaftr query $@ > $TEST_OUTPUT_FNAME + local lineno=`cat $TEST_OUTPUT_FNAME | wc -l` if [[ $lineno -eq 1 ]]; then - echo "Success: lwaftr query $*" + echo "Success: lwaftr query no counters $*" else - fatal "lwaftr query $*" + cat $TEST_OUTPUT_FNAME + exit_on_error "Error: lwaftr query no counters $*" fi } diff --git a/src/program/lwaftr/query/tests/test_query.sh b/src/program/lwaftr/query/tests/test_query.sh index bbbb283311..fdca2b7f8f 100755 --- a/src/program/lwaftr/query/tests/test_query.sh +++ b/src/program/lwaftr/query/tests/test_query.sh @@ -1,26 +1,28 @@ #!/usr/bin/env bash -TEST_NAME="query" +TEST_NAME="lwaftr query" # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh +source ${TEST_DIR}/common.sh || exit $? -check_for_root -check_nics_available $TEST_NAME +check_nics_available "$TEST_NAME" +check_commands_available "$TEST_NAME" tmux # QUERY_TEST_DIR is also set by the caller. -source ${QUERY_TEST_DIR}/test_env.sh +source ${QUERY_TEST_DIR}/test_env.sh || exit $? echo "Testing ${TEST_NAME}" -trap cleanup EXIT HUP INT QUIT TERM +trap query_cleanup EXIT HUP INT QUIT TERM LWAFTR_NAME=lwaftr-$$ LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf -# Run lwAFTR. -tmux_launch "lwaftr" "./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" "lwaftr.log" +# Launch "lwaftr run". +CMD_LINE="./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF" +CMD_LINE+=" --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" +tmux_launch "$CMD_LINE" "lwaftr.log" sleep 2 # Test query all. @@ -35,6 +37,9 @@ if [[ -n "$pid" ]]; then fi # Test query by name. -test_lwaftr_query "--name $LWAFTR_NAME" -test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" -test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" +## FIXME: currently broken in non-reconfigurable mode. +#test_lwaftr_query "--name $LWAFTR_NAME" +#test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" +#test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" + +exit 0 diff --git a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh index 484bb89709..090d61be94 100755 --- a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh +++ b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh @@ -1,26 +1,29 @@ #!/usr/bin/env bash -TEST_NAME="query-reconfigurable" +TEST_NAME="lwaftr query --reconfigurable" # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh +source ${TEST_DIR}/common.sh || exit $? -check_for_root -check_nics_available $TEST_NAME +check_nics_available "$TEST_NAME" +check_commands_available "$TEST_NAME" tmux # QUERY_TEST_DIR is also set by the caller. -source ${QUERY_TEST_DIR}/test_env.sh +source ${QUERY_TEST_DIR}/test_env.sh || exit $? echo "Testing ${TEST_NAME}" -trap cleanup EXIT HUP INT QUIT TERM +trap query_cleanup EXIT HUP INT QUIT TERM LWAFTR_NAME=lwaftr-$$ LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf -# Run lwAFTR. -tmux_launch "lwaftr" "./snabb lwaftr run --reconfigurable --name $LWAFTR_NAME --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" "lwaftr.log" +# Launch "lwaftr run". Do not break the "lwaftr run --reconfigurable" string, +# see get_lwaftr_leader in common.sh . +CMD_LINE="./snabb lwaftr run --reconfigurable --name $LWAFTR_NAME" +CMD_LINE+=" --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" +tmux_launch "$CMD_LINE" "lwaftr.log" sleep 2 # Test query all. @@ -46,3 +49,5 @@ fi test_lwaftr_query "--name $LWAFTR_NAME" test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" + +exit 0 diff --git a/src/program/lwaftr/tests/common.sh b/src/program/lwaftr/tests/common.sh index e832343569..3721adefd8 100755 --- a/src/program/lwaftr/tests/common.sh +++ b/src/program/lwaftr/tests/common.sh @@ -4,8 +4,8 @@ SKIPPED_CODE=43 # Show $1 as error message and exit with code $2, or 1 if not passed. function exit_on_error { - (>&2 echo $1) - if [[ -n "$2" ]]; then + (>&2 echo "$1") + if [[ -n $2 ]]; then exit $2 else exit 1 @@ -13,18 +13,21 @@ function exit_on_error { } # Check that the script is run as root, otherwise exit. -function check_for_root { - if [[ $EUID != 0 ]]; then - exit_on_error "This script must be run as root" - fi -} +if [[ $EUID != 0 ]]; then + exit_on_error "Tests must be run as root, exiting." +fi -# Check that a command is available, otherwise exit with code $SKIPPED_CODE. -function check_command_available { - which "$1" &> /dev/null - if [[ $? -ne 0 ]]; then - exit_on_error "No $1 tool present, unable to run test." $SKIPPED_CODE - fi +# If one of the commands from $2 onward is not available, exit +# with code $SKIPPED_CODE mentioning the test name passed in $1. +function check_commands_available { + local test_name=$1 + shift + for cmd in $@; do + which $cmd &> /dev/null + if [[ $? -ne 0 ]]; then + exit_on_error "Cannot find $cmd, skipping $test_name" $SKIPPED_CODE + fi + done } # Check that NIC interfaces are available, otherwise exit with code $SKIPPED_CODE. @@ -69,3 +72,16 @@ function assert_equal { fi fi } + +# Launch a command in a new window of an existing session, or in a new session. +# First parameter: the command as one string; +# second parameter: the log file name. +function tmux_launch { + local command="$1 2>&1 | tee $2" + if [ -z "$tmux_session" ]; then + tmux_session=test_env-$$ + tmux new-session -d -n "lwaftr" -s $tmux_session "$command" + else + tmux new-window -a -d -n "lwaftr" -t $tmux_session "$command" + fi +} diff --git a/src/program/lwaftr/tests/config/selftest.sh b/src/program/lwaftr/tests/config/selftest.sh index 8f9d9d2c74..821e516e6c 100755 --- a/src/program/lwaftr/tests/config/selftest.sh +++ b/src/program/lwaftr/tests/config/selftest.sh @@ -4,9 +4,9 @@ export TEST_DIR="./program/lwaftr/tests" export CONFIG_TEST_DIR=${TEST_DIR}/config -${CONFIG_TEST_DIR}/test-config-get.sh -${CONFIG_TEST_DIR}/test-config-set.sh -${CONFIG_TEST_DIR}/test-config-add.sh -${CONFIG_TEST_DIR}/test-config-remove.sh -${CONFIG_TEST_DIR}/test-config-get-state.sh -${CONFIG_TEST_DIR}/test-config-listen.sh +${CONFIG_TEST_DIR}/test-config-get.sh || exit $? +${CONFIG_TEST_DIR}/test-config-set.sh || exit $? +${CONFIG_TEST_DIR}/test-config-add.sh || exit $? +${CONFIG_TEST_DIR}/test-config-remove.sh || exit $? +${CONFIG_TEST_DIR}/test-config-get-state.sh || exit $? +${CONFIG_TEST_DIR}/test-config-listen.sh || exit $? diff --git a/src/program/lwaftr/tests/config/test-config-add.sh b/src/program/lwaftr/tests/config/test-config-add.sh index d0c154466d..7825c93630 100755 --- a/src/program/lwaftr/tests/config/test-config-add.sh +++ b/src/program/lwaftr/tests/config/test-config-add.sh @@ -2,19 +2,19 @@ ## This adds a softwire section and then checks it can be got ## back and that all the values are as they should be. +TEST_NAME="config add" + # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh - -check_for_root +source ${TEST_DIR}/common.sh || exit $? # CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh +source ${CONFIG_TEST_DIR}/test_env.sh || exit $? -echo "Testing config add" +echo "Testing ${TEST_NAME}" # Come up with a name for the lwaftr. -SNABB_NAME="`random_name`" +SNABB_NAME=lwaftr-$$ # Start the bench command. start_lwaftr_bench $SNABB_NAME diff --git a/src/program/lwaftr/tests/config/test-config-get-state.sh b/src/program/lwaftr/tests/config/test-config-get-state.sh index 7e3fc7180c..bbbcff1882 100755 --- a/src/program/lwaftr/tests/config/test-config-get-state.sh +++ b/src/program/lwaftr/tests/config/test-config-get-state.sh @@ -3,19 +3,19 @@ ## that it will run and produce values. The script has no way of ## validating the accuracy of the values, but it'll check it works. +TEST_NAME="config get state" + # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh - -check_for_root +source ${TEST_DIR}/common.sh || exit $? # CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh +source ${CONFIG_TEST_DIR}/test_env.sh || exit $? -echo "Testing config get state" +echo "Testing ${TEST_NAME}" # Come up with a name for the lwaftr. -SNABB_NAME="`random_name`" +SNABB_NAME=lwaftr-$$ # Start the bench command. start_lwaftr_bench $SNABB_NAME diff --git a/src/program/lwaftr/tests/config/test-config-get.sh b/src/program/lwaftr/tests/config/test-config-get.sh index 7c0259b2da..b91500bd20 100755 --- a/src/program/lwaftr/tests/config/test-config-get.sh +++ b/src/program/lwaftr/tests/config/test-config-get.sh @@ -1,22 +1,21 @@ #!/usr/bin/env bash -## This tests querying from a known config. The test is obviously -## dependent on the values in the test data files used, however this -## allows for testing basic "getting". It performs numerous gets -## on different paths. +## Test querying from a known config. The test is dependent on the values +## in the test data files, however this allows for testing basic "getting". +## It performs numerous gets on different paths. + +TEST_NAME="config get" # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh - -check_for_root +source ${TEST_DIR}/common.sh || exit $? # CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh +source ${CONFIG_TEST_DIR}/test_env.sh || exit $? -echo "Testing config get" +echo "Testing ${TEST_NAME}" # Come up with a name for the lwaftr. -SNABB_NAME="`random_name`" +SNABB_NAME=lwaftr-$$ # Start the bench command. start_lwaftr_bench $SNABB_NAME diff --git a/src/program/lwaftr/tests/config/test-config-listen.sh b/src/program/lwaftr/tests/config/test-config-listen.sh index 3cb92c886a..db0da45554 100755 --- a/src/program/lwaftr/tests/config/test-config-listen.sh +++ b/src/program/lwaftr/tests/config/test-config-listen.sh @@ -3,25 +3,24 @@ ## It only tests the socket method of communicating with the listen ## command due to the difficulties of testing interactive scripts. +TEST_NAME="config listen" + # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh - -check_for_root +source ${TEST_DIR}/common.sh || exit $? # Verify we have the "nc" tool, used to communicate with sockets, # and the "python" interpreter available. # If we don't have them, we just have to skip this test. -check_command_available nc -check_command_available python +check_commands_available "$TEST_NAME" nc python # CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh +source ${CONFIG_TEST_DIR}/test_env.sh || exit $? -echo "Testing config listen" +echo "Testing ${TEST_NAME}" # Come up with a name for the lwaftr. -SNABB_NAME="`random_name`" +SNABB_NAME=lwaftr-$$ # Start the bench command. start_lwaftr_bench $SNABB_NAME diff --git a/src/program/lwaftr/tests/config/test-config-remove.sh b/src/program/lwaftr/tests/config/test-config-remove.sh index dfe2bcb845..32cfc6cbc4 100755 --- a/src/program/lwaftr/tests/config/test-config-remove.sh +++ b/src/program/lwaftr/tests/config/test-config-remove.sh @@ -2,19 +2,19 @@ ## This adds a softwire section and then checks it can be got ## back and that all the values are as they should be. +TEST_NAME="config remove" + # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh - -check_for_root +source ${TEST_DIR}/common.sh || exit $? # CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh +source ${CONFIG_TEST_DIR}/test_env.sh || exit $? -echo "Testing config remove" +echo "Testing ${TEST_NAME}" # Come up with a name for the lwaftr. -SNABB_NAME="`random_name`" +SNABB_NAME=lwaftr-$$ # Start the bench command. start_lwaftr_bench $SNABB_NAME diff --git a/src/program/lwaftr/tests/config/test-config-set.sh b/src/program/lwaftr/tests/config/test-config-set.sh index 719730a029..f3813f08fb 100755 --- a/src/program/lwaftr/tests/config/test-config-set.sh +++ b/src/program/lwaftr/tests/config/test-config-set.sh @@ -2,19 +2,19 @@ ## This checks you can set values, it'll then perform a get to ## verify the value set is the value that is got too. +TEST_NAME="config set" + # TEST_DIR is set by the caller, and passed onward. export TEST_DIR -source ${TEST_DIR}/common.sh - -check_for_root +source ${TEST_DIR}/common.sh || exit $? # CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh +source ${CONFIG_TEST_DIR}/test_env.sh || exit $? -echo "Testing config set" +echo "Testing ${TEST_NAME}" # Come up with a name for the lwaftr. -SNABB_NAME="`random_name`" +SNABB_NAME=lwaftr-$$ # Start the bench command. start_lwaftr_bench $SNABB_NAME @@ -45,5 +45,4 @@ IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-ta PSID="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" assert_equal "$PSID" "$TEST_PSID" -# Stop the lwaftr process. stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test_env.sh b/src/program/lwaftr/tests/config/test_env.sh index 69e6fa3a45..234199c856 100755 --- a/src/program/lwaftr/tests/config/test_env.sh +++ b/src/program/lwaftr/tests/config/test_env.sh @@ -1,23 +1,21 @@ #!/usr/bin/env bash -function random_name { - cat /dev/urandom | tr -dc 'a-z' | fold -w 20 | head -n 1 -} - # TEST_DIR is set by the caller. +source ${TEST_DIR}/common.sh || exit $? + DATA_DIR="${TEST_DIR}/data" BENCHDATA_DIR="${TEST_DIR}/benchdata" -# Start the lwaftr process. The process should end when the script ends; -# however, if something goes wrong and it doesn't end correctly, -# a duration is set to prevent it running indefinitely. +# Start the "lwaftr bench" process. The first parameter is the command name. +# The process should end when the script ends; however, if something goes wrong +# and it doesn't end correctly, a duration is set to prevent it running indefinitely. function start_lwaftr_bench { ./snabb lwaftr bench --reconfigurable --bench-file /dev/null --name "$1" \ --duration 30 \ program/lwaftr/tests/data/icmp_on_fail.conf \ program/lwaftr/tests/benchdata/ipv{4,6}-0550.pcap &> /dev/null & - # Not ideal, but it takes a little time for the lwaftr to properly start. + # It takes a little time for lwaftr to properly start. sleep 2 } From 76b6b19e8fe3b82803498302fbeac5cf5016f280 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 20 Feb 2017 17:54:23 +0100 Subject: [PATCH 562/631] Stop using tmux in query tests, use shell jobs instead (#746) --- src/program/lwaftr/query/tests/test_query.sh | 6 ++---- .../lwaftr/query/tests/test_query_reconfigurable.sh | 9 +++------ src/program/lwaftr/tests/common.sh | 13 ------------- src/program/lwaftr/tests/config/test_env.sh | 8 ++++---- 4 files changed, 9 insertions(+), 27 deletions(-) diff --git a/src/program/lwaftr/query/tests/test_query.sh b/src/program/lwaftr/query/tests/test_query.sh index fdca2b7f8f..b6bfd3bfcb 100755 --- a/src/program/lwaftr/query/tests/test_query.sh +++ b/src/program/lwaftr/query/tests/test_query.sh @@ -7,7 +7,6 @@ export TEST_DIR source ${TEST_DIR}/common.sh || exit $? check_nics_available "$TEST_NAME" -check_commands_available "$TEST_NAME" tmux # QUERY_TEST_DIR is also set by the caller. source ${QUERY_TEST_DIR}/test_env.sh || exit $? @@ -20,9 +19,8 @@ LWAFTR_NAME=lwaftr-$$ LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf # Launch "lwaftr run". -CMD_LINE="./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF" -CMD_LINE+=" --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" -tmux_launch "$CMD_LINE" "lwaftr.log" +./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF \ + --v4 $SNABB_PCI0 --v6 $SNABB_PCI1 &> lwaftr.log & sleep 2 # Test query all. diff --git a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh index 090d61be94..288fc9a66d 100755 --- a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh +++ b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh @@ -7,7 +7,6 @@ export TEST_DIR source ${TEST_DIR}/common.sh || exit $? check_nics_available "$TEST_NAME" -check_commands_available "$TEST_NAME" tmux # QUERY_TEST_DIR is also set by the caller. source ${QUERY_TEST_DIR}/test_env.sh || exit $? @@ -19,11 +18,9 @@ trap query_cleanup EXIT HUP INT QUIT TERM LWAFTR_NAME=lwaftr-$$ LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf -# Launch "lwaftr run". Do not break the "lwaftr run --reconfigurable" string, -# see get_lwaftr_leader in common.sh . -CMD_LINE="./snabb lwaftr run --reconfigurable --name $LWAFTR_NAME" -CMD_LINE+=" --conf $LWAFTR_CONF --v4 $SNABB_PCI0 --v6 $SNABB_PCI1" -tmux_launch "$CMD_LINE" "lwaftr.log" +# Launch "lwaftr run". +./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF --reconfigurable \ + --v4 $SNABB_PCI0 --v6 $SNABB_PCI1 &> lwaftr.log & sleep 2 # Test query all. diff --git a/src/program/lwaftr/tests/common.sh b/src/program/lwaftr/tests/common.sh index 3721adefd8..6366f2517b 100755 --- a/src/program/lwaftr/tests/common.sh +++ b/src/program/lwaftr/tests/common.sh @@ -72,16 +72,3 @@ function assert_equal { fi fi } - -# Launch a command in a new window of an existing session, or in a new session. -# First parameter: the command as one string; -# second parameter: the log file name. -function tmux_launch { - local command="$1 2>&1 | tee $2" - if [ -z "$tmux_session" ]; then - tmux_session=test_env-$$ - tmux new-session -d -n "lwaftr" -s $tmux_session "$command" - else - tmux new-window -a -d -n "lwaftr" -t $tmux_session "$command" - fi -} diff --git a/src/program/lwaftr/tests/config/test_env.sh b/src/program/lwaftr/tests/config/test_env.sh index 234199c856..4905950848 100755 --- a/src/program/lwaftr/tests/config/test_env.sh +++ b/src/program/lwaftr/tests/config/test_env.sh @@ -21,20 +21,20 @@ function start_lwaftr_bench { function stop_lwaftr_bench { # Get the job number for lwaftr bench. - local jobid="`jobs | grep -i \"lwaftr bench\" | awk '{print $1}' | tr -d '[]+'`" - kill -15 "%$jobid" + local jobid=`jobs | grep -i "lwaftr bench" | awk '{print $1}' | tr -d '[]+'` + kill "%$jobid" # Wait until it's shutdown. wait &> /dev/null } function stop_if_running { # Check if it's running, if not, job done. - kill -0 "$1" &> /dev/null + kill -0 $1 &> /dev/null if [[ $? -ne 0 ]]; then return fi # It's running, try and close it nicely. - kill -15 "$1" + kill $1 wait &> /dev/null } From 9d15cdc83eb26d05a6605a6adb13622c38e9cd8f Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 21 Feb 2017 19:12:08 +0100 Subject: [PATCH 563/631] Fix #716 - allow / in yang strings --- src/lib/yang/parser.lua | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 5874037e99..9b39e511c3 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -76,6 +76,14 @@ function Parser:next() return chr end +function Parser:peek_n(n) + local end_index = self.pos + n - 1 + if end_index < #self.str then + return self:peek() .. self.str:sub(self.pos, (end_index - 1)) + end + return self:peek() .. self.str:sub(self.pos) +end + function Parser:check(expected) if self:peek() == expected then if expected then self:next() end @@ -84,6 +92,13 @@ function Parser:check(expected) return false end +function Parser:check_patten(patten) + if not self:is_eof() then + return self:peek():match(patten) + end + return false +end + function Parser:consume(expected) if not self:check(expected) then local ch = self:peek() @@ -189,7 +204,19 @@ end function Parser:parse_string() if self:check("'") then return self:parse_qstring("'") elseif self:check('"') then return self:parse_qstring('"') - else return self:take_while("[^%s;{}\"'/]") end + else + local ret = "" + repeat + ret = ret .. self:take_while("[^%s;{}\"'/]") + if self:is_eof() then break end + if self:peek_n(2) == "/*" then break end + if self:peek() == "/" then + ret = ret .. "/" + self:consume("/") + end + until not self:check_patten("[^%s;{}\"'/]") + return ret + end end function Parser:parse_identifier() @@ -401,7 +428,8 @@ function selftest() argument="port", statements={{keyword="type", argument="number"}}}}) test_module(lines("leaf port {", "type;", "}"), {{keyword="leaf", argument="port", statements={{keyword="type"}}}}) - + test_module('description hello/world;', {{keyword="description", + argument="hello/world"}}) parse(require('lib.yang.ietf_inet_types_yang')) parse(require('lib.yang.ietf_yang_types_yang')) parse(require('lib.yang.ietf_softwire_yang')) From 836ad56bc5c8669d2532a90b9e572d032bab619c Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 22 Feb 2017 16:38:45 +0100 Subject: [PATCH 564/631] #716 - Fix small nits brought up in review --- src/lib/yang/parser.lua | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 9b39e511c3..c19194f3fd 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -92,7 +92,7 @@ function Parser:check(expected) return false end -function Parser:check_patten(patten) +function Parser:check_pattern(patten) if not self:is_eof() then return self:peek():match(patten) end @@ -210,11 +210,10 @@ function Parser:parse_string() ret = ret .. self:take_while("[^%s;{}\"'/]") if self:is_eof() then break end if self:peek_n(2) == "/*" then break end - if self:peek() == "/" then + if self:check("/") then ret = ret .. "/" - self:consume("/") end - until not self:check_patten("[^%s;{}\"'/]") + until not self:check_pattern("[^%s;{}\"'/]") return ret end end From 92ac9244051f2140ece86e8d26cec720b56f8e56 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 22 Feb 2017 17:06:33 +0100 Subject: [PATCH 565/631] #716 - Fix a few more nits --- src/lib/yang/parser.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index c19194f3fd..7bdb861809 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -92,9 +92,9 @@ function Parser:check(expected) return false end -function Parser:check_pattern(patten) +function Parser:check_pattern(pattern) if not self:is_eof() then - return self:peek():match(patten) + return self:peek():match(pattern) end return false end @@ -205,16 +205,16 @@ function Parser:parse_string() if self:check("'") then return self:parse_qstring("'") elseif self:check('"') then return self:parse_qstring('"') else - local ret = "" + local ret = {} repeat - ret = ret .. self:take_while("[^%s;{}\"'/]") + table.insert(ret, self:take_while("[^%s;{}\"'/]")) if self:is_eof() then break end if self:peek_n(2) == "/*" then break end if self:check("/") then - ret = ret .. "/" + table.insert(ret, "/") end until not self:check_pattern("[^%s;{}\"'/]") - return ret + return table.concat(ret) end end From a57d7e374eb19263fc95c305bfdae5293d53d2e6 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 22 Feb 2017 17:38:33 +0100 Subject: [PATCH 566/631] #716 - Add // comment handling in strings Add // comment handling in the middle of unquoted strings and a test to check for this situation. --- src/lib/yang/parser.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/yang/parser.lua b/src/lib/yang/parser.lua index 7bdb861809..db7bf5a2d0 100644 --- a/src/lib/yang/parser.lua +++ b/src/lib/yang/parser.lua @@ -210,6 +210,7 @@ function Parser:parse_string() table.insert(ret, self:take_while("[^%s;{}\"'/]")) if self:is_eof() then break end if self:peek_n(2) == "/*" then break end + if self:peek_n(2) == "//" then break end if self:check("/") then table.insert(ret, "/") end @@ -410,6 +411,7 @@ function selftest() test_string('"// foo bar;"', '// foo bar;') test_string('"/* foo bar */"', '/* foo bar */') test_string([["foo \"bar\""]], 'foo "bar"') + test_string("hello//world", "hello") test_string(lines(" 'foo", " bar'"), lines("foo", " bar")) test_string(lines(" 'foo", " bar'"), lines("foo", "bar")) test_string(lines(" 'foo", "\tbar'"), lines("foo", " bar")) From 080410199d5df8c2d51fe548cb1f02f13de6ed57 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 27 Feb 2017 10:15:38 +0100 Subject: [PATCH 567/631] Add v. 1.12.9 of the 'sh' Python library (#750) --- src/program/lwaftr/tests/lib/__init__.py | 0 src/program/lwaftr/tests/lib/sh.py | 3399 ++++++++++++++++++++++ 2 files changed, 3399 insertions(+) create mode 100644 src/program/lwaftr/tests/lib/__init__.py create mode 100644 src/program/lwaftr/tests/lib/sh.py diff --git a/src/program/lwaftr/tests/lib/__init__.py b/src/program/lwaftr/tests/lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/program/lwaftr/tests/lib/sh.py b/src/program/lwaftr/tests/lib/sh.py new file mode 100644 index 0000000000..780e7613e6 --- /dev/null +++ b/src/program/lwaftr/tests/lib/sh.py @@ -0,0 +1,3399 @@ +""" +http://amoffat.github.io/sh/ +""" +#=============================================================================== +# Copyright (C) 2011-2017 by Andrew Moffat +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +#=============================================================================== + + +__version__ = "1.12.9" +__project_url__ = "https://github.com/amoffat/sh" + + +import platform + +if "windows" in platform.system().lower(): # pragma: no cover + raise ImportError("sh %s is currently only supported on linux and osx. \ +please install pbs 0.110 (http://pypi.python.org/pypi/pbs) for windows \ +support." % __version__) + + +import sys +IS_PY3 = sys.version_info[0] == 3 +MINOR_VER = sys.version_info[1] +IS_PY26 = sys.version_info[0] == 2 and MINOR_VER == 6 + +import traceback +import os +import re +import time +import getpass +from types import ModuleType, GeneratorType +from functools import partial +import inspect +import tempfile +import stat +import glob as glob_module +import ast +from contextlib import contextmanager +import pwd +import errno +from io import UnsupportedOperation + +from locale import getpreferredencoding +DEFAULT_ENCODING = getpreferredencoding() or "UTF-8" + +# normally i would hate this idea of using a global to signify whether we are +# running tests, because it breaks the assumption that what is running in the +# tests is what will run live, but we ONLY use this in a place that has no +# serious side-effects that could change anything. as long as we do that, it +# should be ok +RUNNING_TESTS = bool(int(os.environ.get("SH_TESTS_RUNNING", "0"))) + +if IS_PY3: + from io import StringIO + ioStringIO = StringIO + from io import BytesIO as cStringIO + iocStringIO = cStringIO + from queue import Queue, Empty + + # for some reason, python 3.1 removed the builtin "callable", wtf + if not hasattr(__builtins__, "callable"): + def callable(ob): + return hasattr(ob, "__call__") +else: + from StringIO import StringIO + from cStringIO import OutputType as cStringIO + from io import StringIO as ioStringIO + from io import BytesIO as iocStringIO + from Queue import Queue, Empty + +IS_OSX = platform.system() == "Darwin" +THIS_DIR = os.path.dirname(os.path.realpath(__file__)) +SH_LOGGER_NAME = __name__ + + +import errno +import pty +import termios +import signal +import gc +import select +import threading +import tty +import fcntl +import struct +import resource +from collections import deque +import logging +import weakref + + +# a re-entrant lock for pushd. this way, multiple threads that happen to use +# pushd will all see the current working directory for the duration of the +# with-context +PUSHD_LOCK = threading.RLock() + + +if hasattr(inspect, "signature"): + def get_num_args(fn): + return len(inspect.signature(fn).parameters) +else: + def get_num_args(fn): + return len(inspect.getargspec(fn).args) + +if IS_PY3: + raw_input = input + unicode = str + basestring = str + long = int + + +_unicode_methods = set(dir(unicode())) + + +def encode_to_py3bytes_or_py2str(s): + """ takes anything and attempts to return a py2 string or py3 bytes. this + is typically used when creating command + arguments to be executed via + os.exec* """ + + fallback_encoding = "utf8" + + if IS_PY3: + # if we're already bytes, do nothing + if isinstance(s, bytes): + pass + else: + s = str(s) + try: + s = bytes(s, DEFAULT_ENCODING) + except UnicodeEncodeError: + s = bytes(s, fallback_encoding) + else: + # attempt to convert the thing to unicode from the system's encoding + try: + s = unicode(s, DEFAULT_ENCODING) + # if the thing is already unicode, or it's a number, it can't be + # coerced to unicode with an encoding argument, but if we leave out + # the encoding argument, it will convert it to a string, then to unicode + except TypeError: + s = unicode(s) + + # now that we have guaranteed unicode, encode to our system encoding, + # but attempt to fall back to something + try: + s = s.encode(DEFAULT_ENCODING) + except: + s = s.encode(fallback_encoding, "replace") + return s + + +def _indent_text(text, num=4): + lines = [] + for line in text.split("\n"): + line = (" " * num) + line + lines.append(line) + return "\n".join(lines) + + +class ForkException(Exception): + def __init__(self, orig_exc): + tmpl = """ + +Original exception: +=================== + +%s +""" + msg = tmpl % _indent_text(orig_exc) + Exception.__init__(self, msg) + + +class ErrorReturnCodeMeta(type): + """ a metaclass which provides the ability for an ErrorReturnCode (or + derived) instance, imported from one sh module, to be considered the + subclass of ErrorReturnCode from another module. this is mostly necessary + in the tests, where we do assertRaises, but the ErrorReturnCode that the + program we're testing throws may not be the same class that we pass to + assertRaises + """ + def __subclasscheck__(self, o): + other_bases = set([b.__name__ for b in o.__bases__]) + return self.__name__ in other_bases or o.__name__ == self.__name__ + + +class ErrorReturnCode(Exception): + __metaclass__ = ErrorReturnCodeMeta + + """ base class for all exceptions as a result of a command's exit status + being deemed an error. this base class is dynamically subclassed into + derived classes with the format: ErrorReturnCode_NNN where NNN is the exit + code number. the reason for this is it reduces boiler plate code when + testing error return codes: + + try: + some_cmd() + except ErrorReturnCode_12: + print("couldn't do X") + + vs: + try: + some_cmd() + except ErrorReturnCode as e: + if e.exit_code == 12: + print("couldn't do X") + + it's not much of a savings, but i believe it makes the code easier to read """ + + truncate_cap = 750 + + def __init__(self, full_cmd, stdout, stderr, truncate=True): + self.full_cmd = full_cmd + self.stdout = stdout + self.stderr = stderr + + exc_stdout = self.stdout + if truncate: + exc_stdout = exc_stdout[:self.truncate_cap] + out_delta = len(self.stdout) - len(exc_stdout) + if out_delta: + exc_stdout += ("... (%d more, please see e.stdout)" % out_delta).encode() + + exc_stderr = self.stderr + if truncate: + exc_stderr = exc_stderr[:self.truncate_cap] + err_delta = len(self.stderr) - len(exc_stderr) + if err_delta: + exc_stderr += ("... (%d more, please see e.stderr)" % err_delta).encode() + + msg_tmpl = unicode("\n\n RAN: {cmd}\n\n STDOUT:\n{stdout}\n\n STDERR:\n{stderr}") + + msg = msg_tmpl.format( + cmd=self.full_cmd, + stdout=exc_stdout.decode(DEFAULT_ENCODING, "replace"), + stderr=exc_stderr.decode(DEFAULT_ENCODING, "replace") + ) + + super(ErrorReturnCode, self).__init__(msg) + + +class SignalException(ErrorReturnCode): pass +class TimeoutException(Exception): + """ the exception thrown when a command is killed because a specified + timeout (via _timeout) was hit """ + def __init__(self, exit_code): + self.exit_code = exit_code + super(Exception, self).__init__() + +SIGNALS_THAT_SHOULD_THROW_EXCEPTION = ( + signal.SIGABRT, + signal.SIGBUS, + signal.SIGFPE, + signal.SIGILL, + signal.SIGINT, + signal.SIGKILL, + signal.SIGPIPE, + signal.SIGQUIT, + signal.SIGSEGV, + signal.SIGTERM, + signal.SIGSYS, +) + + +# we subclass AttributeError because: +# https://github.com/ipython/ipython/issues/2577 +# https://github.com/amoffat/sh/issues/97#issuecomment-10610629 +class CommandNotFound(AttributeError): pass + + + + +rc_exc_regex = re.compile("(ErrorReturnCode|SignalException)_((\d+)|SIG[a-zA-Z]+)") +rc_exc_cache = {} + +SIGNAL_MAPPING = {} +for k,v in signal.__dict__.items(): + if re.match(r"SIG[a-zA-Z]+", k): + SIGNAL_MAPPING[v] = k + + +def get_exc_from_name(name): + """ takes an exception name, like: + + ErrorReturnCode_1 + SignalException_9 + SignalException_SIGHUP + + and returns the corresponding exception. this is primarily used for + importing exceptions from sh into user code, for instance, to capture those + exceptions """ + + exc = None + try: + return rc_exc_cache[name] + except KeyError: + m = rc_exc_regex.match(name) + if m: + base = m.group(1) + rc_or_sig_name = m.group(2) + + if base == "SignalException": + try: + rc = -int(rc_or_sig_name) + except ValueError: + rc = -getattr(signal, rc_or_sig_name) + else: + rc = int(rc_or_sig_name) + + exc = get_rc_exc(rc) + return exc + + +def get_rc_exc(rc): + """ takes a exit code or negative signal number and produces an exception + that corresponds to that return code. positive return codes yield + ErrorReturnCode exception, negative return codes yield SignalException + + we also cache the generated exception so that only one signal of that type + exists, preserving identity """ + + try: + return rc_exc_cache[rc] + except KeyError: + pass + + if rc > 0: + name = "ErrorReturnCode_%d" % rc + base = ErrorReturnCode + else: + signame = SIGNAL_MAPPING[abs(rc)] + name = "SignalException_" + signame + base = SignalException + + exc = ErrorReturnCodeMeta(name, (base,), {"exit_code": rc}) + rc_exc_cache[rc] = exc + return exc + + + +# we monkey patch glob. i'm normally generally against monkey patching, but i +# decided to do this really un-intrusive patch because we need a way to detect +# if a list that we pass into an sh command was generated from glob. the reason +# being that glob returns an empty list if a pattern is not found, and so +# commands will treat the empty list as no arguments, which can be a problem, +# ie: +# +# ls(glob("*.ojfawe")) +# +# ^ will show the contents of your home directory, because it's essentially +# running ls([]) which, as a process, is just "ls". +# +# so we subclass list and monkey patch the glob function. nobody should be the +# wiser, but we'll have results that we can make some determinations on +_old_glob = glob_module.glob + +class GlobResults(list): + def __init__(self, path, results): + self.path = path + list.__init__(self, results) + +def glob(path, *args, **kwargs): + expanded = GlobResults(path, _old_glob(path, *args, **kwargs)) + return expanded + +glob_module.glob = glob + + + + +def which(program, paths=None): + """ takes a program name or full path, plus an optional collection of search + paths, and returns the full path of the requested executable. if paths is + specified, it is the entire list of search paths, and the PATH env is not + used at all. otherwise, PATH env is used to look for the program """ + + def is_exe(fpath): + return (os.path.exists(fpath) and + os.access(fpath, os.X_OK) and + os.path.isfile(os.path.realpath(fpath))) + + found_path = None + fpath, fname = os.path.split(program) + + # if there's a path component, then we've specified a path to the program, + # and we should just test if that program is executable. if it is, return + if fpath: + if is_exe(program): + found_path = program + + # otherwise, we've just passed in the program name, and we need to search + # the paths to find where it actually lives + else: + paths_to_search = [] + + if isinstance(paths, (tuple, list)): + paths_to_search.extend(paths) + else: + env_paths = os.environ.get("PATH", "").split(os.pathsep) + paths_to_search.extend(env_paths) + + for path in paths_to_search: + exe_file = os.path.join(path, program) + if is_exe(exe_file): + found_path = exe_file + break + + return found_path + + +def resolve_command_path(program): + path = which(program) + if not path: + # our actual command might have a dash in it, but we can't call + # that from python (we have to use underscores), so we'll check + # if a dash version of our underscore command exists and use that + # if it does + if "_" in program: + path = which(program.replace("_", "-")) + if not path: + return None + return path + + +def resolve_command(name, baked_args=None): + path = resolve_command_path(name) + cmd = None + if path: + cmd = Command(path) + if baked_args: + cmd = cmd.bake(**baked_args) + return cmd + + + + +class Logger(object): + """ provides a memory-inexpensive logger. a gotcha about python's builtin + logger is that logger objects are never garbage collected. if you create a + thousand loggers with unique names, they'll sit there in memory until your + script is done. with sh, it's easy to create loggers with unique names if + we want our loggers to include our command arguments. for example, these + are all unique loggers: + + ls -l + ls -l /tmp + ls /tmp + + so instead of creating unique loggers, and without sacrificing logging + output, we use this class, which maintains as part of its state, the logging + "context", which will be the very unique name. this allows us to get a + logger with a very general name, eg: "command", and have a unique name + appended to it via the context, eg: "ls -l /tmp" """ + def __init__(self, name, context=None): + self.name = name + self.log = logging.getLogger("%s.%s" % (SH_LOGGER_NAME, name)) + self.set_context(context) + + def _format_msg(self, msg, *args): + if self.context: + msg = "%s: %s" % (self.context, msg) + return msg % args + + def set_context(self, context): + if context: + context = context.replace("%", "%%") + self.context = context or "" + + def get_child(self, name, context): + new_name = self.name + "." + name + new_context = self.context + "." + context + l = Logger(new_name, new_context) + return l + + def info(self, msg, *args): + self.log.info(self._format_msg(msg, *args)) + + def debug(self, msg, *args): + self.log.debug(self._format_msg(msg, *args)) + + def error(self, msg, *args): + self.log.error(self._format_msg(msg, *args)) + + def exception(self, msg, *args): + self.log.exception(self._format_msg(msg, *args)) + + +def default_logger_str(cmd, call_args, pid=None): + if pid: + s = "" % (cmd, pid) + else: + s = "" % cmd + return s + + + +class RunningCommand(object): + """ this represents an executing Command object. it is returned as the + result of __call__() being executed on a Command instance. this creates a + reference to a OProc instance, which is a low-level wrapper around the + process that was exec'd + + this is the class that gets manipulated the most by user code, and so it + implements various convenience methods and logical mechanisms for the + underlying process. for example, if a user tries to access a + backgrounded-process's stdout/err, the RunningCommand object is smart enough + to know to wait() on the process to finish first. and when the process + finishes, RunningCommand is smart enough to translate exit codes to + exceptions. """ + + # these are attributes that we allow to passthrough to OProc for + _OProc_attr_whitelist = set(( + "signal", + "terminate", + "kill", + "kill_group", + "signal_group", + "pid", + "sid", + "pgid", + "ctty", + + "input_thread_exc", + "output_thread_exc", + "bg_thread_exc", + )) + + def __init__(self, cmd, call_args, stdin, stdout, stderr): + """ + cmd is an array, where each element is encoded as bytes (PY3) or str + (PY2) + """ + + # self.ran is used for auditing what actually ran. for example, in + # exceptions, or if you just want to know what was ran after the + # command ran + # + # here we're making a consistent unicode string out if our cmd. + # we're also assuming (correctly, i think) that the command and its + # arguments are the encoding we pass into _encoding, which falls back to + # the system's encoding + enc = call_args["encoding"] + self.ran = " ".join([arg.decode(enc, "ignore") for arg in cmd]) + + self.call_args = call_args + self.cmd = cmd + + self.process = None + self._process_completed = False + should_wait = True + spawn_process = True + + # this is used to track if we've already raised StopIteration, and if we + # have, raise it immediately again if the user tries to call next() on + # us. https://github.com/amoffat/sh/issues/273 + self._stopped_iteration = False + + # with contexts shouldn't run at all yet, they prepend + # to every command in the context + if call_args["with"]: + spawn_process = False + get_prepend_stack().append(self) + + + if call_args["piped"] or call_args["iter"] or call_args["iter_noblock"]: + should_wait = False + + # we're running in the background, return self and let us lazily + # evaluate + if call_args["bg"]: + should_wait = False + + # redirection + if call_args["err_to_out"]: + stderr = OProc.STDOUT + + done_callback = call_args["done"] + if done_callback: + call_args["done"] = partial(done_callback, self) + + + # set up which stream should write to the pipe + # TODO, make pipe None by default and limit the size of the Queue + # in oproc.OProc + pipe = OProc.STDOUT + if call_args["iter"] == "out" or call_args["iter"] is True: + pipe = OProc.STDOUT + elif call_args["iter"] == "err": + pipe = OProc.STDERR + + if call_args["iter_noblock"] == "out" or call_args["iter_noblock"] is True: + pipe = OProc.STDOUT + elif call_args["iter_noblock"] == "err": + pipe = OProc.STDERR + + # there's currently only one case where we wouldn't spawn a child + # process, and that's if we're using a with-context with our command + self._spawned_and_waited = False + if spawn_process: + log_str_factory = call_args["log_msg"] or default_logger_str + logger_str = log_str_factory(self.ran, call_args) + self.log = Logger("command", logger_str) + + self.log.info("starting process") + + if should_wait: + self._spawned_and_waited = True + + # this lock is needed because of a race condition where a background + # thread, created in the OProc constructor, may try to access + # self.process, but it has not been assigned yet + process_assign_lock = threading.Lock() + with process_assign_lock: + self.process = OProc(self, self.log, cmd, stdin, stdout, stderr, + self.call_args, pipe, process_assign_lock) + + logger_str = log_str_factory(self.ran, call_args, self.process.pid) + self.log.set_context(logger_str) + self.log.info("process started") + + if should_wait: + self.wait() + + + def wait(self): + """ waits for the running command to finish. this is called on all + running commands, eventually, except for ones that run in the background + """ + if not self._process_completed: + self._process_completed = True + + exit_code = self.process.wait() + if self.process.timed_out: + # if we timed out, our exit code represents a signal, which is + # negative, so let's make it positive to store in our + # TimeoutException + raise TimeoutException(-exit_code) + + else: + self.handle_command_exit_code(exit_code) + + # if an iterable command is using an instance of OProc for its stdin, + # wait on it. the process is probably set to "piped", which means it + # won't be waited on, which means exceptions won't propagate up to the + # main thread. this allows them to bubble up + if self.process._stdin_process: + self.process._stdin_process.command.wait() + + self.log.info("process completed") + return self + + + def handle_command_exit_code(self, code): + """ here we determine if we had an exception, or an error code that we + weren't expecting to see. if we did, we create and raise an exception + """ + exc_class = get_exc_exit_code_would_raise(code, self.call_args["ok_code"]) + if exc_class: + exc = exc_class(self.ran, self.process.stdout, self.process.stderr, + self.call_args["truncate_exc"]) + raise exc + + + @property + def stdout(self): + self.wait() + return self.process.stdout + + @property + def stderr(self): + self.wait() + return self.process.stderr + + @property + def exit_code(self): + self.wait() + return self.process.exit_code + + + def __len__(self): + return len(str(self)) + + def __enter__(self): + """ we don't actually do anything here because anything that should have + been done would have been done in the Command.__call__ call. + essentially all that has to happen is the comand be pushed on the + prepend stack. """ + pass + + def __iter__(self): + return self + + def next(self): + """ allow us to iterate over the output of our command """ + + if self._stopped_iteration: + raise StopIteration() + + # we do this because if get blocks, we can't catch a KeyboardInterrupt + # so the slight timeout allows for that. + while True: + try: + chunk = self.process._pipe_queue.get(True, 0.001) + except Empty: + if self.call_args["iter_noblock"]: + return errno.EWOULDBLOCK + else: + if chunk is None: + self.wait() + self._stopped_iteration = True + raise StopIteration() + try: + return chunk.decode(self.call_args["encoding"], + self.call_args["decode_errors"]) + except UnicodeDecodeError: + return chunk + + + # python 3 + __next__ = next + + def __exit__(self, typ, value, traceback): + if self.call_args["with"] and get_prepend_stack(): + get_prepend_stack().pop() + + def __str__(self): + """ in python3, should return unicode. in python2, should return a + string of bytes """ + if IS_PY3: + return self.__unicode__() + else: + return unicode(self).encode(self.call_args["encoding"]) + + def __unicode__(self): + """ a magic method defined for python2. calling unicode() on a + RunningCommand object will call this """ + if self.process and self.stdout: + return self.stdout.decode(self.call_args["encoding"], + self.call_args["decode_errors"]) + elif IS_PY3: + return "" + else: + return unicode("") + + def __eq__(self, other): + return unicode(self) == unicode(other) + __hash__ = None # Avoid DeprecationWarning in Python < 3 + + def __contains__(self, item): + return item in str(self) + + def __getattr__(self, p): + # let these three attributes pass through to the OProc object + if p in self._OProc_attr_whitelist: + if self.process: + return getattr(self.process, p) + else: + raise AttributeError + + # see if strings have what we're looking for. we're looking at the + # method names explicitly because we don't want to evaluate self unless + # we absolutely have to, the reason being, in python2, hasattr swallows + # exceptions, and if we try to run hasattr on a command that failed and + # is being run with _iter=True, the command will be evaluated, throw an + # exception, but hasattr will discard it + if p in _unicode_methods: + return getattr(unicode(self), p) + + raise AttributeError + + def __repr__(self): + """ in python3, should return unicode. in python2, should return a + string of bytes """ + try: + return str(self) + except UnicodeDecodeError: + if self.process: + if self.stdout: + return repr(self.stdout) + return repr("") + + def __long__(self): + return long(str(self).strip()) + + def __float__(self): + return float(str(self).strip()) + + def __int__(self): + return int(str(self).strip()) + + + +def output_redirect_is_filename(out): + return isinstance(out, basestring) + + +def get_prepend_stack(): + tl = Command.thread_local + if not hasattr(tl, "_prepend_stack"): + tl._prepend_stack = [] + return tl._prepend_stack + + +def special_kwarg_validator(kwargs, invalid_list): + s1 = set(kwargs.keys()) + invalid_args = [] + + for args in invalid_list: + + if callable(args): + fn = args + ret = fn(kwargs) + invalid_args.extend(ret) + + else: + args, error_msg = args + + if s1.issuperset(args): + invalid_args.append((args, error_msg)) + + return invalid_args + + +def get_fileno(ob): + # in py2, this will return None. in py3, it will return an method that + # raises when called + fileno_meth = getattr(ob, "fileno", None) + + fileno = None + if fileno_meth: + # py3 StringIO objects will report a fileno, but calling it will raise + # an exception + try: + fileno = fileno_meth() + except UnsupportedOperation: + pass + elif isinstance(ob, (int,long)) and ob >= 0: + fileno = ob + + return fileno + + +def ob_is_tty(ob): + """ checks if an object (like a file-like object) is a tty. """ + fileno = get_fileno(ob) + is_tty = False + if fileno: + is_tty = os.isatty(fileno) + return is_tty + +def ob_is_pipe(ob): + fileno = get_fileno(ob) + is_pipe = False + if fileno: + fd_stat = os.fstat(fileno) + is_pipe = stat.S_ISFIFO(fd_stat.st_mode) + return is_pipe + + +def tty_in_validator(kwargs): + pairs = (("tty_in", "in"), ("tty_out", "out")) + invalid = [] + for tty, std in pairs: + if tty in kwargs and ob_is_tty(kwargs.get(std, None)): + args = (tty, std) + error = "`_%s` is a TTY already, so so it doesn't make sense \ +to set up a TTY with `_%s`" % (std, tty) + invalid.append((args, error)) + + return invalid + +def bufsize_validator(kwargs): + """ a validator to prevent a user from saying that they want custom + buffering when they're using an in/out object that will be os.dup'd to the + process, and has its own buffering. an example is a pipe or a tty. it + doesn't make sense to tell them to have a custom buffering, since the os + controls this. """ + invalid = [] + + in_ob = kwargs.get("in", None) + out_ob = kwargs.get("out", None) + + in_buf = kwargs.get("in_bufsize", None) + out_buf = kwargs.get("out_bufsize", None) + + in_no_buf = ob_is_tty(in_ob) or ob_is_pipe(in_ob) + out_no_buf = ob_is_tty(out_ob) or ob_is_pipe(out_ob) + + err = "Can't specify an {target} bufsize if the {target} target is a pipe or TTY" + + if in_no_buf and in_buf is not None: + invalid.append((("in", "in_bufsize"), err.format(target="in"))) + + if out_no_buf and out_buf is not None: + invalid.append((("out", "out_bufsize"), err.format(target="out"))) + + return invalid + + +class Command(object): + """ represents an un-run system program, like "ls" or "cd". because it + represents the program itself (and not a running instance of it), it should + hold very little state. in fact, the only state it does hold is baked + arguments. + + when a Command object is called, the result that is returned is a + RunningCommand object, which represents the Command put into an execution + state. """ + thread_local = threading.local() + + _call_args = { + "fg": False, # run command in foreground + + # run a command in the background. commands run in the background + # ignore SIGHUP and do not automatically exit when the parent process + # ends + "bg": False, + + # automatically report exceptions for background commands + "bg_exc": True, + + "with": False, # prepend the command to every command after it + "in": None, + "out": None, # redirect STDOUT + "err": None, # redirect STDERR + "err_to_out": None, # redirect STDERR to STDOUT + + # stdin buffer size + # 1 for line, 0 for unbuffered, any other number for that amount + "in_bufsize": 0, + # stdout buffer size, same values as above + "out_bufsize": 1, + "err_bufsize": 1, + + # this is how big the output buffers will be for stdout and stderr. + # this is essentially how much output they will store from the process. + # we use a deque, so if it overflows past this amount, the first items + # get pushed off as each new item gets added. + # + # NOTICE + # this is not a *BYTE* size, this is a *CHUNK* size...meaning, that if + # you're buffering out/err at 1024 bytes, the internal buffer size will + # be "internal_bufsize" CHUNKS of 1024 bytes + "internal_bufsize": 3 * 1024 ** 2, + + "env": None, + "piped": None, + "iter": None, + "iter_noblock": None, + "ok_code": 0, + "cwd": None, + + # the separator delimiting between a long-argument's name and its value + # setting this to None will cause name and value to be two separate + # arguments, like for short options + # for example, --arg=derp, '=' is the long_sep + "long_sep": "=", + + # the prefix used for long arguments + "long_prefix": "--", + + # this is for programs that expect their input to be from a terminal. + # ssh is one of those programs + "tty_in": False, + "tty_out": True, + + "encoding": DEFAULT_ENCODING, + "decode_errors": "strict", + + # how long the process should run before it is auto-killed + "timeout": None, + "timeout_signal": signal.SIGKILL, + + # TODO write some docs on "long-running processes" + # these control whether or not stdout/err will get aggregated together + # as the process runs. this has memory usage implications, so sometimes + # with long-running processes with a lot of data, it makes sense to + # set these to true + "no_out": False, + "no_err": False, + "no_pipe": False, + + # if any redirection is used for stdout or stderr, internal buffering + # of that data is not stored. this forces it to be stored, as if + # the output is being T'd to both the redirected destination and our + # internal buffers + "tee": None, + + # will be called when a process terminates regardless of exception + "done": None, + + # a tuple (rows, columns) of the desired size of both the stdout and + # stdin ttys, if ttys are being used + "tty_size": (20, 80), + + # whether or not our exceptions should be truncated + "truncate_exc": True, + + # a function to call after the child forks but before the process execs + "preexec_fn": None, + + # UID to set after forking. Requires root privileges. Not supported on + # Windows. + "uid": None, + + # put the forked process in its own process session? + "new_session": True, + + # pre-process args passed into __call__. only really useful when used + # in .bake() + "arg_preprocess": None, + + # a callable that produces a log message from an argument tuple of the + # command and the args + "log_msg": None, + } + + # this is a collection of validators to make sure the special kwargs make + # sense + _kwarg_validators = ( + (("fg", "bg"), "Command can't be run in the foreground and background"), + (("fg", "err_to_out"), "Can't redirect STDERR in foreground mode"), + (("err", "err_to_out"), "Stderr is already being redirected"), + (("piped", "iter"), "You cannot iterate when this command is being piped"), + (("piped", "no_pipe"), "Using a pipe doesn't make sense if you've \ +disabled the pipe"), + (("no_out", "iter"), "You cannot iterate over output if there is no \ +output"), + tty_in_validator, + bufsize_validator, + ) + + + def __init__(self, path, search_paths=None): + found = which(path, search_paths) + + self._path = encode_to_py3bytes_or_py2str("") + + # is the command baked (aka, partially applied)? + self._partial = False + self._partial_baked_args = [] + self._partial_call_args = {} + + # bugfix for functools.wraps. issue #121 + self.__name__ = str(self) + + if not found: + raise CommandNotFound(path) + + # the reason why we set the values early in the constructor, and again + # here, is for people who have tools that inspect the stack on + # exception. if CommandNotFound is raised, we need self._path and the + # other attributes to be set correctly, so repr() works when they're + # inspecting the stack. issue #304 + self._path = encode_to_py3bytes_or_py2str(found) + self.__name__ = str(self) + + + def __getattribute__(self, name): + # convenience + getattr = partial(object.__getattribute__, self) + val = None + + if name.startswith("_"): + val = getattr(name) + + elif name == "bake": + val = getattr("bake") + + # here we have a way of getting past shadowed subcommands. for example, + # if "git bake" was a thing, we wouldn't be able to do `git.bake()` + # because `.bake()` is already a method. so we allow `git.bake_()` + elif name.endswith("_"): + name = name[:-1] + + if val is None: + val = getattr("bake")(name) + + return val + + + @staticmethod + def _extract_call_args(kwargs): + """ takes kwargs that were passed to a command's __call__ and extracts + out the special keyword arguments, we return a tuple of special keyword + args, and kwargs that will go to the execd command """ + + kwargs = kwargs.copy() + call_args = {} + for parg, default in Command._call_args.items(): + key = "_" + parg + + if key in kwargs: + call_args[parg] = kwargs[key] + del kwargs[key] + + invalid_kwargs = special_kwarg_validator(call_args, + Command._kwarg_validators) + + if invalid_kwargs: + exc_msg = [] + for args, error_msg in invalid_kwargs: + exc_msg.append(" %r: %s" % (args, error_msg)) + exc_msg = "\n".join(exc_msg) + raise TypeError("Invalid special arguments:\n\n%s\n" % exc_msg) + + return call_args, kwargs + + + # TODO needs documentation + def bake(self, *args, **kwargs): + fn = type(self)(self._path) + fn._partial = True + + call_args, kwargs = self._extract_call_args(kwargs) + + pruned_call_args = call_args + for k, v in Command._call_args.items(): + try: + if pruned_call_args[k] == v: + del pruned_call_args[k] + except KeyError: + continue + + fn._partial_call_args.update(self._partial_call_args) + fn._partial_call_args.update(pruned_call_args) + fn._partial_baked_args.extend(self._partial_baked_args) + sep = pruned_call_args.get("long_sep", self._call_args["long_sep"]) + prefix = pruned_call_args.get("long_prefix", + self._call_args["long_prefix"]) + fn._partial_baked_args.extend(compile_args(args, kwargs, sep, prefix)) + return fn + + def __str__(self): + """ in python3, should return unicode. in python2, should return a + string of bytes """ + if IS_PY3: + return self.__unicode__() + else: + return self.__unicode__().encode(DEFAULT_ENCODING) + + + def __eq__(self, other): + return str(self) == str(other) + + __hash__ = None # Avoid DeprecationWarning in Python < 3 + + + def __repr__(self): + """ in python3, should return unicode. in python2, should return a + string of bytes """ + return "" % str(self) + + + def __unicode__(self): + """ a magic method defined for python2. calling unicode() on a + self will call this """ + baked_args = " ".join(item.decode(DEFAULT_ENCODING) for item in self._partial_baked_args) + if baked_args: + baked_args = " " + baked_args + return self._path.decode(DEFAULT_ENCODING) + baked_args + + def __enter__(self): + self(_with=True) + + def __exit__(self, typ, value, traceback): + get_prepend_stack().pop() + + + def __call__(self, *args, **kwargs): + + kwargs = kwargs.copy() + args = list(args) + + # this will hold our final command, including arguments, that will be + # execd + cmd = [] + + # this will hold a complete mapping of all our special keyword arguments + # and their values + call_args = Command._call_args.copy() + + # aggregate any 'with' contexts + for prepend in get_prepend_stack(): + pcall_args = prepend.call_args.copy() + # don't pass the 'with' call arg + pcall_args.pop("with", None) + + call_args.update(pcall_args) + cmd.extend(prepend.cmd) + + cmd.append(self._path) + + # do we have an argument pre-processor? if so, run it. we need to do + # this early, so that args, kwargs are accurate + preprocessor = self._partial_call_args.get("arg_preprocess", None) + if preprocessor: + args, kwargs = preprocessor(args, kwargs) + + # here we extract the special kwargs and override any + # special kwargs from the possibly baked command + extracted_call_args, kwargs = self._extract_call_args(kwargs) + + call_args.update(self._partial_call_args) + call_args.update(extracted_call_args) + + + # handle a None. this is added back only to not break the api in the + # 1.* version. TODO remove this in 2.0, as "ok_code", if specified, + # should always be a definitive value or list of values, and None is + # ambiguous + if call_args["ok_code"] is None: + call_args["ok_code"] = 0 + + if not getattr(call_args["ok_code"], "__iter__", None): + call_args["ok_code"] = [call_args["ok_code"]] + + + # check if we're piping via composition + stdin = call_args["in"] + if args: + first_arg = args.pop(0) + if isinstance(first_arg, RunningCommand): + if first_arg.call_args["piped"]: + stdin = first_arg.process + else: + stdin = first_arg.process._pipe_queue + + else: + args.insert(0, first_arg) + + processed_args = compile_args(args, kwargs, call_args["long_sep"], + call_args["long_prefix"]) + + # makes sure our arguments are broken up correctly + split_args = self._partial_baked_args + processed_args + + final_args = split_args + + cmd.extend(final_args) + + # if we're running in foreground mode, we need to completely bypass + # launching a RunningCommand and OProc and just do a spawn + if call_args["fg"]: + if call_args["env"] is None: + launch = lambda: os.spawnv(os.P_WAIT, cmd[0], cmd) + else: + launch = lambda: os.spawnve(os.P_WAIT, cmd[0], cmd, call_args["env"]) + + exit_code = launch() + exc_class = get_exc_exit_code_would_raise(exit_code, call_args["ok_code"]) + if exc_class: + if IS_PY3: + ran = " ".join([arg.decode(DEFAULT_ENCODING, "ignore") for arg in cmd]) + else: + ran = " ".join(cmd) + exc = exc_class(ran, b"", b"", call_args["truncate_exc"]) + raise exc + return None + + + # stdout redirection + stdout = call_args["out"] + if output_redirect_is_filename(stdout): + stdout = open(str(stdout), "wb") + + # stderr redirection + stderr = call_args["err"] + if output_redirect_is_filename(stderr): + stderr = open(str(stderr), "wb") + + return RunningCommand(cmd, call_args, stdin, stdout, stderr) + + +def compile_args(args, kwargs, sep, prefix): + """ takes args and kwargs, as they were passed into the command instance + being executed with __call__, and compose them into a flat list that + will eventually be fed into exec. example: + + with this call: + + sh.ls("-l", "/tmp", color="never") + + this function receives + + args = ['-l', '/tmp'] + kwargs = {'color': 'never'} + + and produces + + ['-l', '/tmp', '--color=never'] + + """ + processed_args = [] + encode = encode_to_py3bytes_or_py2str + + # aggregate positional args + for arg in args: + if isinstance(arg, (list, tuple)): + if isinstance(arg, GlobResults) and not arg: + arg = [arg.path] + + for sub_arg in arg: + processed_args.append(encode(sub_arg)) + elif isinstance(arg, dict): + processed_args += aggregate_keywords(arg, sep, prefix, raw=True) + else: + processed_args.append(encode(arg)) + + # aggregate the keyword arguments + processed_args += aggregate_keywords(kwargs, sep, prefix) + + return processed_args + + +def aggregate_keywords(keywords, sep, prefix, raw=False): + """ take our keyword arguments, and a separator, and compose the list of + flat long (and short) arguments. example + + {'color': 'never', 't': True, 'something': True} with sep '=' + + becomes + + ['--color=never', '-t', '--something'] + + the `raw` argument indicates whether or not we should leave the argument + name alone, or whether we should replace "_" with "-". if we pass in a + dictionary, like this: + + sh.command({"some_option": 12}) + + then `raw` gets set to True, because we want to leave the key as-is, to + produce: + + ['--some_option=12'] + + but if we just use a command's kwargs, `raw` is False, which means this: + + sh.command(some_option=12) + + becomes: + + ['--some-option=12'] + + eessentially, using kwargs is a convenience, but it lacks the ability to + put a '-' in the name, so we do the replacement of '_' to '-' for you. + but when you really don't want that to happen, you should use a + dictionary instead with the exact names you want + """ + + processed = [] + encode = encode_to_py3bytes_or_py2str + + for k, v in keywords.items(): + # we're passing a short arg as a kwarg, example: + # cut(d="\t") + if len(k) == 1: + if v is not False: + processed.append(encode("-" + k)) + if v is not True: + processed.append(encode(v)) + + # we're doing a long arg + else: + if not raw: + k = k.replace("_", "-") + + if v is True: + processed.append(encode("--" + k)) + elif v is False: + pass + elif sep is None or sep == " ": + processed.append(encode(prefix + k)) + processed.append(encode(v)) + else: + arg = encode("%s%s%s%s" % (prefix, k, sep, v)) + processed.append(arg) + + return processed + + +def _start_daemon_thread(fn, name, exc_queue, *args): + def wrap(*args, **kwargs): + try: + fn(*args, **kwargs) + except Exception as e: + exc_queue.put(e) + raise + + thrd = threading.Thread(target=wrap, name=name, args=args) + thrd.daemon = True + thrd.start() + return thrd + + +def setwinsize(fd, rows_cols): + """ set the terminal size of a tty file descriptor. borrowed logic + from pexpect.py """ + rows, cols = rows_cols + TIOCSWINSZ = getattr(termios, 'TIOCSWINSZ', -2146929561) + + s = struct.pack('HHHH', rows, cols, 0, 0) + fcntl.ioctl(fd, TIOCSWINSZ, s) + +def construct_streamreader_callback(process, handler): + """ here we're constructing a closure for our streamreader callback. this + is used in the case that we pass a callback into _out or _err, meaning we + want to our callback to handle each bit of output + + we construct the closure based on how many arguments it takes. the reason + for this is to make it as easy as possible for people to use, without + limiting them. a new user will assume the callback takes 1 argument (the + data). as they get more advanced, they may want to terminate the process, + or pass some stdin back, and will realize that they can pass a callback of + more args """ + + + # implied arg refers to the "self" that methods will pass in. we need to + # account for this implied arg when figuring out what function the user + # passed in based on number of args + implied_arg = 0 + + partial_args = 0 + handler_to_inspect = handler + + if isinstance(handler, partial): + partial_args = len(handler.args) + handler_to_inspect = handler.func + + if inspect.ismethod(handler_to_inspect): + implied_arg = 1 + num_args = get_num_args(handler_to_inspect) + + else: + if inspect.isfunction(handler_to_inspect): + num_args = get_num_args(handler_to_inspect) + + # is an object instance with __call__ method + else: + implied_arg = 1 + num_args = get_num_args(handler_to_inspect.__call__) + + + net_args = num_args - implied_arg - partial_args + + handler_args = () + + # just the chunk + if net_args == 1: + handler_args = () + + # chunk, stdin + if net_args == 2: + handler_args = (process.stdin,) + + # chunk, stdin, process + elif net_args == 3: + # notice we're only storing a weakref, to prevent cyclic references + # (where the process holds a streamreader, and a streamreader holds a + # handler-closure with a reference to the process + handler_args = (process.stdin, weakref.ref(process)) + + def fn(chunk): + # this is pretty ugly, but we're evaluating the process at call-time, + # because it's a weakref + args = handler_args + if len(args) == 2: + args = (handler_args[0], handler_args[1]()) + return handler(chunk, *args) + + return fn + + +def get_exc_exit_code_would_raise(exit_code, ok_codes): + exc = None + success = exit_code in ok_codes + bad_sig = -exit_code in SIGNALS_THAT_SHOULD_THROW_EXCEPTION + + if not success or bad_sig: + exc = get_rc_exc(exit_code) + return exc + + +def handle_process_exit_code(exit_code): + """ this should only ever be called once for each child process """ + # if we exited from a signal, let our exit code reflect that + if os.WIFSIGNALED(exit_code): + exit_code = -os.WTERMSIG(exit_code) + # otherwise just give us a normal exit code + elif os.WIFEXITED(exit_code): + exit_code = os.WEXITSTATUS(exit_code) + else: + raise RuntimeError("Unknown child exit status!") + + return exit_code + + +def no_interrupt(syscall, *args, **kwargs): + """ a helper for making system calls immune to EINTR """ + ret = None + + while True: + try: + ret = syscall(*args, **kwargs) + except OSError as e: + if e.errno == errno.EINTR: + continue + else: + raise + else: + break + + return ret + + +class OProc(object): + """ this class is instantiated by RunningCommand for a command to be exec'd. + it handles all the nasty business involved with correctly setting up the + input/output to the child process. it gets its name for subprocess.Popen + (process open) but we're calling ours OProc (open process) """ + + _default_window_size = (24, 80) + + # used in redirecting + STDOUT = -1 + STDERR = -2 + + def __init__(self, command, parent_log, cmd, stdin, stdout, stderr, + call_args, pipe, process_assign_lock): + """ + cmd is the full string that will be exec'd. it includes the program + name and all its arguments + + stdin, stdout, stderr are what the child will use for standard + input/output/err + + call_args is a mapping of all the special keyword arguments to apply + to the child process + """ + self.command = command + self.call_args = call_args + + # convenience + ca = self.call_args + + if ca["uid"] is not None: + if os.getuid() != 0: + raise RuntimeError("UID setting requires root privileges") + + target_uid = ca["uid"] + + pwrec = pwd.getpwuid(ca["uid"]) + target_gid = pwrec.pw_gid + + # I had issues with getting 'Input/Output error reading stdin' from dd, + # until I set _tty_out=False + if ca["piped"]: + ca["tty_out"] = False + + self._stdin_process = None + + + # if the objects that we are passing to the OProc happen to be a + # file-like object that is a tty, for example `sys.stdin`, then, later + # on in this constructor, we're going to skip out on setting up pipes + # and pseudoterminals for those endpoints + stdin_is_tty_or_pipe = ob_is_tty(stdin) or ob_is_pipe(stdin) + stdout_is_tty_or_pipe = ob_is_tty(stdout) or ob_is_pipe(stdout) + stderr_is_tty_or_pipe = ob_is_tty(stderr) or ob_is_pipe(stderr) + + # if we're passing in a custom stdout/out/err value, we obviously have + # to force not using single_tty + custom_in_out_err = stdin or stdout or stderr + + single_tty = (ca["tty_in"] and ca["tty_out"]) and not custom_in_out_err + + # this logic is a little convoluted, but basically this top-level + # if/else is for consolidating input and output TTYs into a single + # TTY. this is the only way some secure programs like ssh will + # output correctly (is if stdout and stdin are both the same TTY) + if single_tty: + self._stdin_read_fd, self._stdin_write_fd = pty.openpty() + + self._stdout_read_fd = os.dup(self._stdin_read_fd) + self._stdout_write_fd = os.dup(self._stdin_write_fd) + + self._stderr_read_fd = os.dup(self._stdin_read_fd) + self._stderr_write_fd = os.dup(self._stdin_write_fd) + + # do not consolidate stdin and stdout. this is the most common use- + # case + else: + # this check here is because we may be doing piping and so our stdin + # might be an instance of OProc + if isinstance(stdin, OProc) and stdin.call_args["piped"]: + self._stdin_write_fd = stdin._pipe_fd + self._stdin_read_fd = None + self._stdin_process = stdin + + elif stdin_is_tty_or_pipe: + self._stdin_write_fd = os.dup(get_fileno(stdin)) + self._stdin_read_fd = None + + elif ca["tty_in"]: + self._stdin_read_fd, self._stdin_write_fd = pty.openpty() + + # tty_in=False is the default + else: + self._stdin_write_fd, self._stdin_read_fd = os.pipe() + + + if stdout_is_tty_or_pipe: + self._stdout_write_fd = os.dup(get_fileno(stdout)) + self._stdout_read_fd = None + + # tty_out=True is the default + elif ca["tty_out"]: + self._stdout_read_fd, self._stdout_write_fd = pty.openpty() + + else: + self._stdout_read_fd, self._stdout_write_fd = os.pipe() + + # unless STDERR is going to STDOUT, it ALWAYS needs to be a pipe, + # and never a PTY. the reason for this is not totally clear to me, + # but it has to do with the fact that if STDERR isn't set as the + # CTTY (because STDOUT is), the STDERR buffer won't always flush + # by the time the process exits, and the data will be lost. + # i've only seen this on OSX. + if stderr is OProc.STDOUT: + self._stderr_read_fd = os.dup(self._stdout_read_fd) + self._stderr_write_fd = os.dup(self._stdout_write_fd) + + elif stderr_is_tty_or_pipe: + self._stderr_write_fd = os.dup(get_fileno(stderr)) + self._stderr_read_fd = None + + else: + self._stderr_read_fd, self._stderr_write_fd = os.pipe() + + + piped = ca["piped"] + self._pipe_fd = None + if piped: + fd_to_use = self._stdout_read_fd + if piped == "err": + fd_to_use = self._stderr_read_fd + self._pipe_fd = os.dup(fd_to_use) + + + new_session = ca["new_session"] + needs_ctty = ca["tty_in"] and new_session + + self.ctty = None + if needs_ctty: + self.ctty = os.ttyname(self._stdin_write_fd) + + # this is a hack, but what we're doing here is intentionally throwing an + # OSError exception if our child processes's directory doesn't exist, + # but we're doing it BEFORE we fork. the reason for before the fork is + # error handling. i'm currently too lazy to implement what + # subprocess.py did and set up a error pipe to handle exceptions that + # happen in the child between fork and exec. it has only been seen in + # the wild for a missing cwd, so we'll handle it here. + cwd = ca["cwd"] + if cwd is not None and not os.path.exists(cwd): + os.chdir(cwd) + + gc_enabled = gc.isenabled() + if gc_enabled: + gc.disable() + + # for synchronizing + session_pipe_read, session_pipe_write = os.pipe() + exc_pipe_read, exc_pipe_write = os.pipe() + + # this pipe is for synchronzing with the child that the parent has + # closed its in/out/err fds. this is a bug on OSX (but not linux), + # where we can lose output sometimes, due to a race, if we do + # os.close(self._stdout_write_fd) in the parent after the child starts + # writing. + if IS_OSX: + close_pipe_read, close_pipe_write = os.pipe() + + + # session id, group id, process id + self.sid = None + self.pgid = None + self.pid = os.fork() + + # child + if self.pid == 0: # pragma: no cover + if IS_OSX: + os.read(close_pipe_read, 1) + os.close(close_pipe_read) + os.close(close_pipe_write) + + try: + # ignoring SIGHUP lets us persist even after the parent process + # exits. only ignore if we're backgrounded + if ca["bg"] is True: + signal.signal(signal.SIGHUP, signal.SIG_IGN) + + # put our forked process in a new session? this will relinquish + # any control of our inherited CTTY and also make our parent + # process init + if new_session: + os.setsid() + # if we're not going in a new session, we should go in a new + # process group. this way, our process, and any children it + # spawns, are alone, contained entirely in one group. if we + # didn't do this, and didn't use a new session, then our exec'd + # process *could* exist in the same group as our python process, + # depending on how we launch the process (from a shell, or some + # other way) + else: + os.setpgrp() + + sid = os.getsid(0) + pgid = os.getpgid(0) + payload = ("%d,%d" % (sid, pgid)).encode(DEFAULT_ENCODING) + os.write(session_pipe_write, payload) + + if ca["tty_out"] and not stdout_is_tty_or_pipe and not single_tty: + # set raw mode, so there isn't any weird translation of + # newlines to \r\n and other oddities. we're not outputting + # to a terminal anyways + # + # we HAVE to do this here, and not in the parent process, + # because we have to guarantee that this is set before the + # child process is run, and we can't do it twice. + tty.setraw(self._stdout_write_fd) + + + # if the parent-side fd for stdin exists, close it. the case + # where it may not exist is if we're using piping + if self._stdin_read_fd: + os.close(self._stdin_read_fd) + + if self._stdout_read_fd: + os.close(self._stdout_read_fd) + + if self._stderr_read_fd: + os.close(self._stderr_read_fd) + + os.close(session_pipe_read) + os.close(exc_pipe_read) + + if cwd: + os.chdir(cwd) + + os.dup2(self._stdin_write_fd, 0) + os.dup2(self._stdout_write_fd, 1) + os.dup2(self._stderr_write_fd, 2) + + + # set our controlling terminal, but only if we're using a tty + # for stdin. it doesn't make sense to have a ctty otherwise + if needs_ctty: + tmp_fd = os.open(os.ttyname(0), os.O_RDWR) + os.close(tmp_fd) + + if ca["tty_out"] and not stdout_is_tty_or_pipe: + setwinsize(1, ca["tty_size"]) + + if ca["uid"] is not None: + os.setgid(target_gid) + os.setuid(target_uid) + + preexec_fn = ca["preexec_fn"] + if callable(preexec_fn): + preexec_fn() + + + # don't inherit file descriptors + max_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + os.closerange(3, max_fd) + + # actually execute the process + if ca["env"] is None: + os.execv(cmd[0], cmd) + else: + os.execve(cmd[0], cmd, ca["env"]) + + # we must ensure that we carefully exit the child process on + # exception, otherwise the parent process code will be executed + # twice on exception https://github.com/amoffat/sh/issues/202 + # + # if your parent process experiences an exit code 255, it is most + # likely that an exception occurred between the fork of the child + # and the exec. this should be reported. + except: + # some helpful debugging + try: + tb = traceback.format_exc().encode("utf8", "ignore") + os.write(exc_pipe_write, tb) + + finally: + os._exit(255) + + # parent + else: + if gc_enabled: + gc.enable() + + os.close(self._stdin_write_fd) + os.close(self._stdout_write_fd) + os.close(self._stderr_write_fd) + + # tell our child process that we've closed our write_fds, so it is + # ok to proceed towards exec. see the comment where this pipe is + # opened, for why this is necessary + if IS_OSX: + os.close(close_pipe_read) + os.write(close_pipe_write, str(1).encode(DEFAULT_ENCODING)) + os.close(close_pipe_write) + + os.close(exc_pipe_write) + fork_exc = os.read(exc_pipe_read, 1024**2) + os.close(exc_pipe_read) + if fork_exc: + fork_exc = fork_exc.decode(DEFAULT_ENCODING) + raise ForkException(fork_exc) + + os.close(session_pipe_write) + sid, pgid = os.read(session_pipe_read, + 1024).decode(DEFAULT_ENCODING).split(",") + os.close(session_pipe_read) + self.sid = int(sid) + self.pgid = int(pgid) + + # used to determine what exception to raise. if our process was + # killed via a timeout counter, we'll raise something different than + # a SIGKILL exception + self.timed_out = False + + self.started = time.time() + self.cmd = cmd + + # exit code should only be manipulated from within self._wait_lock + # to prevent race conditions + self.exit_code = None + + self.stdin = stdin or Queue() + + # _pipe_queue is used internally to hand off stdout from one process + # to another. by default, all stdout from a process gets dumped + # into this pipe queue, to be consumed in real time (hence the + # thread-safe Queue), or at a potentially later time + self._pipe_queue = Queue() + + # this is used to prevent a race condition when we're waiting for + # a process to end, and the OProc's internal threads are also checking + # for the processes's end + self._wait_lock = threading.Lock() + + # these are for aggregating the stdout and stderr. we use a deque + # because we don't want to overflow + self._stdout = deque(maxlen=ca["internal_bufsize"]) + self._stderr = deque(maxlen=ca["internal_bufsize"]) + + if ca["tty_in"] and not stdin_is_tty_or_pipe: + setwinsize(self._stdin_read_fd, ca["tty_size"]) + + + self.log = parent_log.get_child("process", repr(self)) + + + self.log.debug("started process") + + # disable echoing, but only if it's a tty that we created ourselves + if ca["tty_in"] and not stdin_is_tty_or_pipe: + attr = termios.tcgetattr(self._stdin_read_fd) + attr[3] &= ~termios.ECHO + termios.tcsetattr(self._stdin_read_fd, termios.TCSANOW, attr) + + # we're only going to create a stdin thread iff we have potential + # for stdin to come in. this would be through a stdout callback or + # through an object we've passed in for stdin + potentially_has_input = callable(stdout) or stdin + + # this represents the connection from a Queue object (or whatever + # we're using to feed STDIN) to the process's STDIN fd + self._stdin_stream = None + if self._stdin_read_fd and potentially_has_input: + log = self.log.get_child("streamwriter", "stdin") + self._stdin_stream = StreamWriter(log, self._stdin_read_fd, + self.stdin, ca["in_bufsize"], ca["encoding"], + ca["tty_in"]) + + stdout_pipe = None + if pipe is OProc.STDOUT and not ca["no_pipe"]: + stdout_pipe = self._pipe_queue + + + # this represents the connection from a process's STDOUT fd to + # wherever it has to go, sometimes a pipe Queue (that we will use + # to pipe data to other processes), and also an internal deque + # that we use to aggregate all the output + save_stdout = not ca["no_out"] and \ + (ca["tee"] in (True, "out") or stdout is None) + + + pipe_out = ca["piped"] in ("out", True) + pipe_err = ca["piped"] in ("err",) + + # if we're piping directly into another process's filedescriptor, we + # bypass reading from the stdout stream altogether, because we've + # already hooked up this processes's stdout fd to the other + # processes's stdin fd + self._stdout_stream = None + if not pipe_out and self._stdout_read_fd: + if callable(stdout): + stdout = construct_streamreader_callback(self, stdout) + self._stdout_stream = \ + StreamReader( + self.log.get_child("streamreader", "stdout"), + self._stdout_read_fd, stdout, self._stdout, + ca["out_bufsize"], ca["encoding"], + ca["decode_errors"], stdout_pipe, + save_data=save_stdout) + + elif self._stdout_read_fd: + os.close(self._stdout_read_fd) + + + # if stderr is going to one place (because it's grouped with stdout, + # or we're dealing with a single tty), then we don't actually need a + # stream reader for stderr, because we've already set one up for + # stdout above + self._stderr_stream = None + if stderr is not OProc.STDOUT and not single_tty and not pipe_err \ + and self._stderr_read_fd: + + stderr_pipe = None + if pipe is OProc.STDERR and not ca["no_pipe"]: + stderr_pipe = self._pipe_queue + + save_stderr = not ca["no_err"] and \ + (ca["tee"] in ("err",) or stderr is None) + + if callable(stderr): + stderr = construct_streamreader_callback(self, stderr) + + self._stderr_stream = StreamReader(Logger("streamreader"), + self._stderr_read_fd, stderr, self._stderr, + ca["err_bufsize"], ca["encoding"], ca["decode_errors"], + stderr_pipe, save_data=save_stderr) + + elif self._stderr_read_fd: + os.close(self._stderr_read_fd) + + + def timeout_fn(): + self.timed_out = True + self.signal(ca["timeout_signal"]) + + + self._timeout_event = None + self._timeout_timer = None + if ca["timeout"]: + self._timeout_event = threading.Event() + self._timeout_timer = threading.Timer(ca["timeout"], + self._timeout_event.set) + self._timeout_timer.start() + + # this is for cases where we know that the RunningCommand that was + # launched was not .wait()ed on to complete. in those unique cases, + # we allow the thread that processes output to report exceptions in + # that thread. it's important that we only allow reporting of the + # exception, and nothing else (like the additional stuff that + # RunningCommand.wait() does), because we want the exception to be + # re-raised in the future, if we DO call .wait() + handle_exit_code = None + if not self.command._spawned_and_waited and ca["bg_exc"]: + def fn(exit_code): + with process_assign_lock: + return self.command.handle_command_exit_code(exit_code) + handle_exit_code = fn + + self._quit_threads = threading.Event() + + thread_name = "background thread for pid %d" % self.pid + self._bg_thread_exc_queue = Queue(1) + self._background_thread = _start_daemon_thread(background_thread, + thread_name, self._bg_thread_exc_queue, timeout_fn, + self._timeout_event, handle_exit_code, self.is_alive, + self._quit_threads) + + + # start the main io threads. stdin thread is not needed if we are + # connecting from another process's stdout pipe + self._input_thread = None + self._input_thread_exc_queue = Queue(1) + if self._stdin_stream: + close_before_term = not needs_ctty + thread_name = "STDIN thread for pid %d" % self.pid + self._input_thread = _start_daemon_thread(input_thread, + thread_name, self._input_thread_exc_queue, self.log, + self._stdin_stream, self.is_alive, self._quit_threads, + close_before_term) + + + # this event is for cases where the subprocess that we launch + # launches its OWN subprocess and dups the stdout/stderr fds to that + # new subprocess. in that case, stdout and stderr will never EOF, + # so our output_thread will never finish and will hang. this event + # prevents that hanging + self._stop_output_event = threading.Event() + + self._output_thread_exc_queue = Queue(1) + thread_name = "STDOUT/ERR thread for pid %d" % self.pid + self._output_thread = _start_daemon_thread(output_thread, + thread_name, self._output_thread_exc_queue, self.log, + self._stdout_stream, self._stderr_stream, + self._timeout_event, self.is_alive, self._quit_threads, + self._stop_output_event) + + + def __repr__(self): + return "" % (self.pid, self.cmd[:500]) + + + # these next 3 properties are primary for tests + @property + def output_thread_exc(self): + exc = None + try: + exc = self._output_thread_exc_queue.get(False) + except Empty: + pass + return exc + + @property + def input_thread_exc(self): + exc = None + try: + exc = self._input_thread_exc_queue.get(False) + except Empty: + pass + return exc + + @property + def bg_thread_exc(self): + exc = None + try: + exc = self._bg_thread_exc_queue.get(False) + except Empty: + pass + return exc + + + def change_in_bufsize(self, buf): + self._stdin_stream.stream_bufferer.change_buffering(buf) + + def change_out_bufsize(self, buf): + self._stdout_stream.stream_bufferer.change_buffering(buf) + + def change_err_bufsize(self, buf): + self._stderr_stream.stream_bufferer.change_buffering(buf) + + + + @property + def stdout(self): + return "".encode(self.call_args["encoding"]).join(self._stdout) + + @property + def stderr(self): + return "".encode(self.call_args["encoding"]).join(self._stderr) + + def get_pgid(self): + """ return the CURRENT group id of the process. this differs from + self.pgid in that this refects the current state of the process, where + self.pgid is the group id at launch """ + return os.getpgid(self.pid) + + def get_sid(self): + """ return the CURRENT session id of the process. this differs from + self.sid in that this refects the current state of the process, where + self.sid is the session id at launch """ + return os.getsid(self.pid) + + def signal_group(self, sig): + self.log.debug("sending signal %d to group", sig) + os.killpg(self.get_pgid(), sig) + + def signal(self, sig): + self.log.debug("sending signal %d", sig) + os.kill(self.pid, sig) + + def kill_group(self): + self.log.debug("killing group") + self.signal_group(signal.SIGKILL) + + def kill(self): + self.log.debug("killing") + self.signal(signal.SIGKILL) + + def terminate(self): + self.log.debug("terminating") + self.signal(signal.SIGTERM) + + + def is_alive(self): + """ polls if our child process has completed, without blocking. this + method has side-effects, such as setting our exit_code, if we happen to + see our child exit while this is running """ + + if self.exit_code is not None: + return False, self.exit_code + + # what we're doing here essentially is making sure that the main thread + # (or another thread), isn't calling .wait() on the process. because + # .wait() calls os.waitpid(self.pid, 0), we can't do an os.waitpid + # here...because if we did, and the process exited while in this + # thread, the main thread's os.waitpid(self.pid, 0) would raise OSError + # (because the process ended in another thread). + # + # so essentially what we're doing is, using this lock, checking if + # we're calling .wait(), and if we are, let .wait() get the exit code + # and handle the status, otherwise let us do it. + acquired = self._wait_lock.acquire(False) + if not acquired: + if self.exit_code is not None: + return False, self.exit_code + return True, self.exit_code + + try: + # WNOHANG is just that...we're calling waitpid without hanging... + # essentially polling the process. the return result is (0, 0) if + # there's no process status, so we check that pid == self.pid below + # in order to determine how to proceed + pid, exit_code = no_interrupt(os.waitpid, self.pid, os.WNOHANG) + if pid == self.pid: + self.exit_code = handle_process_exit_code(exit_code) + self._process_just_ended() + + return False, self.exit_code + + # no child process + except OSError: + return False, self.exit_code + else: + return True, self.exit_code + finally: + self._wait_lock.release() + + + def _process_just_ended(self): + if self._timeout_timer: + self._timeout_timer.cancel() + + done_callback = self.call_args["done"] + if done_callback: + success = self.exit_code in self.call_args["ok_code"] + done_callback(success, self.exit_code) + + # this can only be closed at the end of the process, because it might be + # the CTTY, and closing it prematurely will send a SIGHUP. we also + # don't want to close it if there's a self._stdin_stream, because that + # is in charge of closing it also + if self._stdin_read_fd and not self._stdin_stream: + os.close(self._stdin_read_fd) + + + def wait(self): + """ waits for the process to complete, handles the exit code """ + + self.log.debug("acquiring wait lock to wait for completion") + # using the lock in a with-context blocks, which is what we want if + # we're running wait() + with self._wait_lock: + self.log.debug("got wait lock") + witnessed_end = False + + if self.exit_code is None: + self.log.debug("exit code not set, waiting on pid") + pid, exit_code = no_interrupt(os.waitpid, self.pid, 0) # blocks + self.exit_code = handle_process_exit_code(exit_code) + witnessed_end = True + + else: + self.log.debug("exit code already set (%d), no need to wait", + self.exit_code) + + self._quit_threads.set() + + # we may not have a thread for stdin, if the pipe has been connected + # via _piped="direct" + if self._input_thread: + self._input_thread.join() + + # wait, then signal to our output thread that the child process is + # done, and we should have finished reading all the stdout/stderr + # data that we can by now + timer = threading.Timer(2.0, self._stop_output_event.set) + timer.start() + + # wait for our stdout and stderr streamreaders to finish reading and + # aggregating the process output + self._output_thread.join() + timer.cancel() + + self._background_thread.join() + + if witnessed_end: + self._process_just_ended() + + return self.exit_code + + + +def input_thread(log, stdin, is_alive, quit, close_before_term): + """ this is run in a separate thread. it writes into our process's + stdin (a streamwriter) and waits the process to end AND everything that + can be written to be written """ + + done = False + closed = False + alive = True + writers = [stdin] + + while writers and alive: + _, to_write, _ = select.select([], writers, [], 1) + + if to_write: + log.debug("%r ready for more input", stdin) + done = stdin.write() + + if done: + writers = [] + if close_before_term: + stdin.close() + closed = True + + alive, _ = is_alive() + + while alive: + quit.wait(1) + alive, _ = is_alive() + + if not closed: + stdin.close() + + +def event_wait(ev, timeout=None): + triggered = ev.wait(timeout) + if IS_PY26: + triggered = ev.is_set() + return triggered + + +def background_thread(timeout_fn, timeout_event, handle_exit_code, is_alive, + quit): + """ handles the timeout logic """ + + # if there's a timeout event, loop + if timeout_event: + while not quit.is_set(): + timed_out = event_wait(timeout_event, 0.1) + if timed_out: + timeout_fn() + break + + # handle_exit_code will be a function ONLY if our command was NOT waited on + # as part of its spawning. in other words, it's probably a background + # command + # + # this reports the exit code exception in our thread. it's purely for the + # user's awareness, and cannot be caught or used in any way, so it's ok to + # suppress this during the tests + if handle_exit_code and not RUNNING_TESTS: # pragma: no cover + alive = True + while alive: + quit.wait(1) + alive, exit_code = is_alive() + + handle_exit_code(exit_code) + + +def output_thread(log, stdout, stderr, timeout_event, is_alive, quit, + stop_output_event): + """ this function is run in a separate thread. it reads from the + process's stdout stream (a streamreader), and waits for it to claim that + its done """ + + readers = [] + errors = [] + + if stdout is not None: + readers.append(stdout) + errors.append(stdout) + if stderr is not None: + readers.append(stderr) + errors.append(stderr) + + # this is our select loop for polling stdout or stderr that is ready to + # be read and processed. if one of those streamreaders indicate that it + # is done altogether being read from, we remove it from our list of + # things to poll. when no more things are left to poll, we leave this + # loop and clean up + while readers: + outputs, inputs, err = no_interrupt(select.select, readers, [], errors, 1) + + # stdout and stderr + for stream in outputs: + log.debug("%r ready to be read from", stream) + done = stream.read() + if done: + readers.remove(stream) + + # for some reason, we have to just ignore streams that have had an + # error. i'm not exactly sure why, but don't remove this until we + # figure that out, and create a test for it + for stream in err: + pass + + if timeout_event and timeout_event.is_set(): + break + + if stop_output_event.is_set(): + break + + # we need to wait until the process is guaranteed dead before closing our + # outputs, otherwise SIGPIPE + alive = True + while alive: + quit.wait(1) + alive, _ = is_alive() + + if stdout: + stdout.close() + + if stderr: + stderr.close() + + +class DoneReadingForever(Exception): pass +class NotYetReadyToRead(Exception): pass + + +def determine_how_to_read_input(input_obj): + """ given some kind of input object, return a function that knows how to + read chunks of that input object. + + each reader function should return a chunk and raise a DoneReadingForever + exception, or return None, when there's no more data to read + + NOTE: the function returned does not need to care much about the requested + buffering type (eg, unbuffered vs newline-buffered). the StreamBufferer + will take care of that. these functions just need to return a + reasonably-sized chunk of data. """ + + get_chunk = None + + if isinstance(input_obj, Queue): + log_msg = "queue" + get_chunk = get_queue_chunk_reader(input_obj) + + elif callable(input_obj): + log_msg = "callable" + get_chunk = get_callable_chunk_reader(input_obj) + + # also handles stringio + elif hasattr(input_obj, "read"): + log_msg = "file descriptor" + get_chunk = get_file_chunk_reader(input_obj) + + elif isinstance(input_obj, basestring): + log_msg = "string" + get_chunk = get_iter_string_reader(input_obj) + + elif isinstance(input_obj, bytes): + log_msg = "bytes" + get_chunk = get_iter_string_reader(input_obj) + + elif isinstance(input_obj, GeneratorType): + log_msg = "generator" + get_chunk = get_iter_chunk_reader(iter(input_obj)) + + else: + try: + it = iter(input_obj) + except TypeError: + raise Exception("unknown input object") + else: + log_msg = "general iterable" + get_chunk = get_iter_chunk_reader(it) + + return get_chunk, log_msg + + + +def get_queue_chunk_reader(stdin): + def fn(): + try: + chunk = stdin.get(True, 0.1) + except Empty: + raise NotYetReadyToRead + if chunk is None: + raise DoneReadingForever + return chunk + return fn + + +def get_callable_chunk_reader(stdin): + def fn(): + try: + data = stdin() + except DoneReadingForever: + raise + + if not data: + raise DoneReadingForever + + return data + + return fn + + +def get_iter_string_reader(stdin): + """ return an iterator that returns a chunk of a string every time it is + called. notice that even though bufsize_type might be line buffered, we're + not doing any line buffering here. that's because our StreamBufferer + handles all buffering. we just need to return a reasonable-sized chunk. """ + bufsize = 1024 + iter_str = (stdin[i:i + bufsize] for i in range(0, len(stdin), bufsize)) + return get_iter_chunk_reader(iter_str) + + +def get_iter_chunk_reader(stdin): + def fn(): + try: + if IS_PY3: + chunk = stdin.__next__() + else: + chunk = stdin.next() + return chunk + except StopIteration: + raise DoneReadingForever + return fn + +def get_file_chunk_reader(stdin): + bufsize = 1024 + + def fn(): + # python 3.* includes a fileno on stringios, but accessing it throws an + # exception. that exception is how we'll know we can't do a select on + # stdin + is_real_file = True + if IS_PY3: + try: + stdin.fileno() + except UnsupportedOperation: + is_real_file = False + + # this select is for files that may not yet be ready to read. we test + # for fileno because StringIO/BytesIO cannot be used in a select + if is_real_file and hasattr(stdin, "fileno"): + outputs, _, _ = select.select([stdin], [], [], 0.1) + if not outputs: + raise NotYetReadyToRead + + chunk = stdin.read(bufsize) + if not chunk: + raise DoneReadingForever + else: + return chunk + + return fn + + +def bufsize_type_to_bufsize(bf_type): + """ for a given bufsize type, return the actual bufsize we will read. + notice that although 1 means "newline-buffered", we're reading a chunk size + of 1024. this is because we have to read something. we let a + StreamBufferer instance handle splitting our chunk on newlines """ + + # newlines + if bf_type == 1: + bufsize = 1024 + # unbuffered + elif bf_type == 0: + bufsize = 1 + # or buffered by specific amount + else: + bufsize = bf_type + + return bufsize + + + +class StreamWriter(object): + """ StreamWriter reads from some input (the stdin param) and writes to a fd + (the stream param). the stdin may be a Queue, a callable, something with + the "read" method, a string, or an iterable """ + + def __init__(self, log, stream, stdin, bufsize_type, encoding, tty_in): + + self.stream = stream + self.stdin = stdin + + self.log = log + self.encoding = encoding + self.tty_in = tty_in + + self.stream_bufferer = StreamBufferer(bufsize_type, self.encoding) + self.get_chunk, log_msg = determine_how_to_read_input(stdin) + self.log.debug("parsed stdin as a %s", log_msg) + + + def fileno(self): + """ defining this allows us to do select.select on an instance of this + class """ + return self.stream + + + + def write(self): + """ attempt to get a chunk of data to write to our child process's + stdin, then write it. the return value answers the questions "are we + done writing forever?" """ + + # get_chunk may sometimes return bytes, and sometimes return strings + # because of the nature of the different types of STDIN objects we + # support + try: + chunk = self.get_chunk() + if chunk is None: + raise DoneReadingForever + + except DoneReadingForever: + self.log.debug("done reading") + + if self.tty_in: + # EOF time + try: + char = termios.tcgetattr(self.stream)[6][termios.VEOF] + except: + char = chr(4).encode() + + # normally, one EOF should be enough to signal to an program + # that is read()ing, to return 0 and be on your way. however, + # some programs are misbehaved, like python3.1 and python3.2. + # they don't stop reading sometimes after read() returns 0. + # this can be demonstrated with the following program: + # + # import sys + # sys.stdout.write(sys.stdin.read()) + # + # then type 'a' followed by ctrl-d 3 times. in python + # 2.6,2.7,3.3,3.4,3.5, it only takes 2 ctrl-d to terminate. + # however, in python 3.1 and 3.2, it takes all 3. + # + # so here we send an extra EOF along, just in case. i don't + # believe it can hurt anything + os.write(self.stream, char) + os.write(self.stream, char) + + return True + + except NotYetReadyToRead: + self.log.debug("received no data") + return False + + # if we're not bytes, make us bytes + if IS_PY3 and hasattr(chunk, "encode"): + chunk = chunk.encode(self.encoding) + + for proc_chunk in self.stream_bufferer.process(chunk): + self.log.debug("got chunk size %d: %r", len(proc_chunk), + proc_chunk[:30]) + + self.log.debug("writing chunk to process") + try: + os.write(self.stream, proc_chunk) + except OSError: + self.log.debug("OSError writing stdin chunk") + return True + + + def close(self): + self.log.debug("closing, but flushing first") + chunk = self.stream_bufferer.flush() + self.log.debug("got chunk size %d to flush: %r", len(chunk), chunk[:30]) + try: + if chunk: + os.write(self.stream, chunk) + + except OSError: + pass + + os.close(self.stream) + + +def determine_how_to_feed_output(handler, encoding, decode_errors): + if callable(handler): + process, finish = get_callback_chunk_consumer(handler, encoding, + decode_errors) + + # in py3, this is used for bytes + elif isinstance(handler, (cStringIO, iocStringIO)): + process, finish = get_cstringio_chunk_consumer(handler) + + # in py3, this is used for unicode + elif isinstance(handler, (StringIO, ioStringIO)): + process, finish = get_stringio_chunk_consumer(handler, encoding, + decode_errors) + + elif hasattr(handler, "write"): + process, finish = get_file_chunk_consumer(handler) + + else: + process = lambda chunk: False + finish = lambda: None + + return process, finish + + +def get_file_chunk_consumer(handler): + encode = lambda chunk: chunk + if getattr(handler, "encoding", None): + encode = lambda chunk: chunk.decode(handler.encoding) + + flush = lambda: None + if hasattr(handler, "flush"): + flush = handler.flush + + def process(chunk): + handler.write(encode(chunk)) + # we should flush on an fd. chunk is already the correctly-buffered + # size, so we don't need the fd buffering as well + flush() + return False + + def finish(): + flush() + + return process, finish + +def get_callback_chunk_consumer(handler, encoding, decode_errors): + def process(chunk): + # try to use the encoding first, if that doesn't work, send + # the bytes, because it might be binary + try: + chunk = chunk.decode(encoding, decode_errors) + except UnicodeDecodeError: + pass + return handler(chunk) + + def finish(): + pass + + return process, finish + +def get_cstringio_chunk_consumer(handler): + def process(chunk): + handler.write(chunk) + return False + + def finish(): + pass + + return process, finish + + +def get_stringio_chunk_consumer(handler, encoding, decode_errors): + def process(chunk): + handler.write(chunk.decode(encoding, decode_errors)) + return False + + def finish(): + pass + + return process, finish + + +class StreamReader(object): + """ reads from some output (the stream) and sends what it just read to the + handler. """ + def __init__(self, log, stream, handler, buffer, bufsize_type, encoding, + decode_errors, pipe_queue=None, save_data=True): + self.stream = stream + self.buffer = buffer + self.save_data = save_data + self.encoding = encoding + self.decode_errors = decode_errors + + self.pipe_queue = None + if pipe_queue: + self.pipe_queue = weakref.ref(pipe_queue) + + self.log = log + + self.stream_bufferer = StreamBufferer(bufsize_type, self.encoding, + self.decode_errors) + self.bufsize = bufsize_type_to_bufsize(bufsize_type) + + self.process_chunk, self.finish_chunk_processor = \ + determine_how_to_feed_output(handler, encoding, decode_errors) + + self.should_quit = False + + + def fileno(self): + """ defining this allows us to do select.select on an instance of this + class """ + return self.stream + + def close(self): + chunk = self.stream_bufferer.flush() + self.log.debug("got chunk size %d to flush: %r", len(chunk), chunk[:30]) + if chunk: + self.write_chunk(chunk) + + self.finish_chunk_processor() + + if self.pipe_queue and self.save_data: + self.pipe_queue().put(None) + + os.close(self.stream) + + + def write_chunk(self, chunk): + # in PY3, the chunk coming in will be bytes, so keep that in mind + + if not self.should_quit: + self.should_quit = self.process_chunk(chunk) + + + if self.save_data: + self.buffer.append(chunk) + + if self.pipe_queue: + self.log.debug("putting chunk onto pipe: %r", chunk[:30]) + self.pipe_queue().put(chunk) + + + def read(self): + # if we're PY3, we're reading bytes, otherwise we're reading + # str + try: + chunk = no_interrupt(os.read, self.stream, self.bufsize) + except OSError as e: + self.log.debug("got errno %d, done reading", e.errno) + return True + if not chunk: + self.log.debug("got no chunk, done reading") + return True + + self.log.debug("got chunk size %d: %r", len(chunk), chunk[:30]) + for chunk in self.stream_bufferer.process(chunk): + self.write_chunk(chunk) + + + + +class StreamBufferer(object): + """ this is used for feeding in chunks of stdout/stderr, and breaking it up + into chunks that will actually be put into the internal buffers. for + example, if you have two processes, one being piped to the other, and you + want that, first process to feed lines of data (instead of the chunks + however they come in), OProc will use an instance of this class to chop up + the data and feed it as lines to be sent down the pipe """ + + def __init__(self, buffer_type, encoding=DEFAULT_ENCODING, + decode_errors="strict"): + # 0 for unbuffered, 1 for line, everything else for that amount + self.type = buffer_type + self.buffer = [] + self.n_buffer_count = 0 + self.encoding = encoding + self.decode_errors = decode_errors + + # this is for if we change buffering types. if we change from line + # buffered to unbuffered, its very possible that our self.buffer list + # has data that was being saved up (while we searched for a newline). + # we need to use that up, so we don't lose it + self._use_up_buffer_first = False + + # the buffering lock is used because we might change the buffering + # types from a different thread. for example, if we have a stdout + # callback, we might use it to change the way stdin buffers. so we + # lock + self._buffering_lock = threading.RLock() + self.log = Logger("stream_bufferer") + + + def change_buffering(self, new_type): + # TODO, when we stop supporting 2.6, make this a with context + self.log.debug("acquiring buffering lock for changing buffering") + self._buffering_lock.acquire() + self.log.debug("got buffering lock for changing buffering") + try: + if new_type == 0: + self._use_up_buffer_first = True + + self.type = new_type + finally: + self._buffering_lock.release() + self.log.debug("released buffering lock for changing buffering") + + + def process(self, chunk): + # MAKE SURE THAT THE INPUT IS PY3 BYTES + # THE OUTPUT IS ALWAYS PY3 BYTES + + # TODO, when we stop supporting 2.6, make this a with context + self.log.debug("acquiring buffering lock to process chunk (buffering: %d)", self.type) + self._buffering_lock.acquire() + self.log.debug("got buffering lock to process chunk (buffering: %d)", self.type) + try: + # unbuffered + if self.type == 0: + if self._use_up_buffer_first: + self._use_up_buffer_first = False + to_write = self.buffer + self.buffer = [] + to_write.append(chunk) + return to_write + + return [chunk] + + # line buffered + elif self.type == 1: + total_to_write = [] + nl = "\n".encode(self.encoding) + while True: + newline = chunk.find(nl) + if newline == -1: + break + + chunk_to_write = chunk[:newline + 1] + if self.buffer: + chunk_to_write = b"".join(self.buffer) + chunk_to_write + + self.buffer = [] + self.n_buffer_count = 0 + + chunk = chunk[newline + 1:] + total_to_write.append(chunk_to_write) + + if chunk: + self.buffer.append(chunk) + self.n_buffer_count += len(chunk) + return total_to_write + + # N size buffered + else: + total_to_write = [] + while True: + overage = self.n_buffer_count + len(chunk) - self.type + if overage >= 0: + ret = "".encode(self.encoding).join(self.buffer) + chunk + chunk_to_write = ret[:self.type] + chunk = ret[self.type:] + total_to_write.append(chunk_to_write) + self.buffer = [] + self.n_buffer_count = 0 + else: + self.buffer.append(chunk) + self.n_buffer_count += len(chunk) + break + return total_to_write + finally: + self._buffering_lock.release() + self.log.debug("released buffering lock for processing chunk (buffering: %d)", self.type) + + + def flush(self): + self.log.debug("acquiring buffering lock for flushing buffer") + self._buffering_lock.acquire() + self.log.debug("got buffering lock for flushing buffer") + try: + ret = "".encode(self.encoding).join(self.buffer) + self.buffer = [] + return ret + finally: + self._buffering_lock.release() + self.log.debug("released buffering lock for flushing buffer") + + + +def with_lock(lock): + def wrapped(fn): + fn = contextmanager(fn) + @contextmanager + def wrapped2(*args, **kwargs): + with lock: + with fn(*args, **kwargs): + yield + return wrapped2 + return wrapped + + +@with_lock(PUSHD_LOCK) +def pushd(path): + """ pushd changes the actual working directory for the duration of the + context, unlike the _cwd arg this will work with other built-ins such as + sh.glob correctly """ + orig_path = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(orig_path) + + +@contextmanager +def args(**kwargs): + """ allows us to temporarily override all the special keyword parameters in + a with context """ + + kwargs_str = ",".join(["%s=%r" % (k,v) for k,v in kwargs.items()]) + + raise DeprecationWarning(""" + +sh.args() has been deprecated because it was never thread safe. use the +following instead: + + sh2 = sh({kwargs}) + sh2.your_command() + +or + + sh2 = sh({kwargs}) + from sh2 import your_command + your_command() + +""".format(kwargs=kwargs_str)) + + + +class Environment(dict): + """ this allows lookups to names that aren't found in the global scope to be + searched for as a program name. for example, if "ls" isn't found in this + module's scope, we consider it a system program and try to find it. + + we use a dict instead of just a regular object as the base class because the + exec() statement used in the run_repl requires the "globals" argument to be a + dictionary """ + + + # this is a list of all of the names that the sh module exports that will + # not resolve to functions. we don't want to accidentally shadow real + # commands with functions/imports that we define in sh.py. for example, + # "import time" may override the time system program + whitelist = set([ + "Command", + "RunningCommand", + "CommandNotFound", + "DEFAULT_ENCODING", + "DoneReadingForever", + "ErrorReturnCode", + "NotYetReadyToRead", + "SignalException", + "ForkException", + "TimeoutException", + "__project_url__", + "__version__", + "__file__", + "args", + "pushd", + "glob", + "contrib", + ]) + + + def __init__(self, globs, baked_args={}): + """ baked_args are defaults for the 'sh' execution context. for + example: + + tmp = sh(_out=StringIO()) + + 'out' would end up in here as an entry in the baked_args dict """ + + self.globs = globs + self.baked_args = baked_args + self.disable_whitelist = False + + def __getitem__(self, k): + # if we first import "_disable_whitelist" from sh, we can import + # anything defined in the global scope of sh.py. this is useful for our + # tests + if k == "_disable_whitelist": + self.disable_whitelist = True + return None + + # we're trying to import something real (maybe), see if it's in our + # global scope + if k in self.whitelist or self.disable_whitelist: + return self.globs[k] + + # somebody tried to be funny and do "from sh import *" + if k == "__all__": + raise RuntimeError("Cannot import * from sh. \ +Please import sh or import programs individually.") + + + # check if we're naming a dynamically generated ReturnCode exception + exc = get_exc_from_name(k) + if exc: + return exc + + + # https://github.com/ipython/ipython/issues/2577 + # https://github.com/amoffat/sh/issues/97#issuecomment-10610629 + if k.startswith("__") and k.endswith("__"): + raise AttributeError + + + # is it a custom builtin? + builtin = getattr(self, "b_" + k, None) + if builtin: + return builtin + + + # is it a command? + cmd = resolve_command(k, self.baked_args) + if cmd: + return cmd + + + # how about an environment variable? + # this check must come after testing if its a command, because on some + # systems, there are an environment variables that can conflict with + # command names. + # https://github.com/amoffat/sh/issues/238 + try: + return os.environ[k] + except KeyError: + pass + + + # nothing found, raise an exception + raise CommandNotFound(k) + + + # methods that begin with "b_" are custom builtins and will override any + # program that exists in our path. this is useful for things like + # common shell builtins that people are used to, but which aren't actually + # full-fledged system binaries + + def b_cd(self, path=None): + if path: + os.chdir(path) + else: + os.chdir(os.path.expanduser('~')) + + def b_which(self, program, paths=None): + return which(program, paths) + + +class Contrib(ModuleType): # pragma: no cover + @classmethod + def __call__(cls, name): + def wrapper1(fn): + + @property + def cmd_getter(self): + cmd = resolve_command(name) + + if not cmd: + raise CommandNotFound(name) + + new_cmd = fn(cmd) + return new_cmd + + setattr(cls, name, cmd_getter) + return fn + + return wrapper1 + + +mod_name = __name__ + ".contrib" +contrib = Contrib(mod_name) +sys.modules[mod_name] = contrib + + +@contrib("git") +def git(orig): # pragma: no cover + """ most git commands play nicer without a TTY """ + cmd = orig.bake(_tty_out=False) + return cmd + +@contrib("sudo") +def sudo(orig): # pragma: no cover + """ a nicer version of sudo that uses getpass to ask for a password, or + allows the first argument to be a string password """ + + prompt = "[sudo] password for %s: " % getpass.getuser() + + def stdin(): + pw = getpass.getpass(prompt=prompt) + "\n" + yield pw + + + def process(args, kwargs): + password = kwargs.pop("password", None) + + if password is None: + pass_getter = stdin() + else: + pass_getter = password.rstrip("\n") + "\n" + + kwargs["_in"] = pass_getter + return args, kwargs + + cmd = orig.bake("-S", _arg_preprocess=process) + return cmd + + + + +def run_repl(env): # pragma: no cover + banner = "\n>> sh v{version}\n>> https://github.com/amoffat/sh\n" + + print(banner.format(version=__version__)) + while True: + try: + line = raw_input("sh> ") + except (ValueError, EOFError): + break + + try: + exec(compile(line, "", "single"), env, env) + except SystemExit: + break + except: + print(traceback.format_exc()) + + # cleans up our last line + print("") + + + + +# this is a thin wrapper around THIS module (we patch sys.modules[__name__]). +# this is in the case that the user does a "from sh import whatever" +# in other words, they only want to import certain programs, not the whole +# system PATH worth of commands. in this case, we just proxy the +# import lookup to our Environment class +class SelfWrapper(ModuleType): + def __init__(self, self_module, baked_args={}): + # this is super ugly to have to copy attributes like this, + # but it seems to be the only way to make reload() behave + # nicely. if i make these attributes dynamic lookups in + # __getattr__, reload sometimes chokes in weird ways... + for attr in ["__builtins__", "__doc__", "__file__", "__name__", "__package__"]: + setattr(self, attr, getattr(self_module, attr, None)) + + # python 3.2 (2.7 and 3.3 work fine) breaks on osx (not ubuntu) + # if we set this to None. and 3.3 needs a value for __path__ + self.__path__ = [] + self.__self_module = self_module + self.__env = Environment(globals(), baked_args=baked_args) + + def __getattr__(self, name): + return self.__env[name] + + def __call__(self, **kwargs): + """ returns a new SelfWrapper object, where all commands spawned from it + have the baked_args kwargs set on them by default """ + baked_args = self.__env.baked_args.copy() + baked_args.update(kwargs) + new_mod = self.__class__(self.__self_module, baked_args) + + # inspect the line in the parent frame that calls and assigns the new sh + # variable, and get the name of the new variable we're assigning to. + # this is very brittle and pretty much a sin. but it works in 99% of + # the time and the tests pass + # + # the reason we need to do this is because we need to remove the old + # cached module from sys.modules. if we don't, it gets re-used, and any + # old baked params get used, which is not what we want + parent = inspect.stack()[1] + code = parent[4][0].strip() + parsed = ast.parse(code) + module_name = parsed.body[0].targets[0].id + + if module_name == __name__: + raise RuntimeError("Cannot use the name 'sh' as an execution context") + + sys.modules.pop(module_name, None) + + return new_mod + + +def in_importlib(frame): + """ helper for checking if a filename is in importlib guts """ + return frame.f_code.co_filename == "" + + +def register_importer(): + """ registers our fancy importer that can let us import from a module name, + like: + + import sh + tmp = sh() + from tmp import ls + """ + + def test(importer): + return importer.__class__.__name__ == ModuleImporterFromVariables.__name__ + already_registered = any([True for i in sys.meta_path if test(i)]) + + if not already_registered: + importer = ModuleImporterFromVariables( + restrict_to=["SelfWrapper"], + ) + sys.meta_path.insert(0, importer) + + return not already_registered + +def fetch_module_from_frame(name, frame): + mod = frame.f_locals.get(name, frame.f_globals.get(name, None)) + return mod + +class ModuleImporterFromVariables(object): + """ a fancy importer that allows us to import from a variable that was + recently set in either the local or global scope, like this: + + sh2 = sh(_timeout=3) + from sh2 import ls + + """ + + def __init__(self, restrict_to=None): + self.restrict_to = set(restrict_to or set()) + + + def find_module(self, mod_fullname, path=None): + """ mod_fullname doubles as the name of the VARIABLE holding our new sh + context. for example: + + derp = sh() + from derp import ls + + here, mod_fullname will be "derp". keep that in mind as we go throug + the rest of this function """ + + parent_frame = inspect.currentframe().f_back + while in_importlib(parent_frame): + parent_frame = parent_frame.f_back + + # this line is saying "hey, does mod_fullname exist as a name we've + # defind previously?" the purpose of this is to ensure that + # mod_fullname is really a thing we've defined. if we haven't defined + # it before, then we "can't" import from it + module = fetch_module_from_frame(mod_fullname, parent_frame) + if not module: + return None + + # make sure it's a class we're allowed to import from + if module.__class__.__name__ not in self.restrict_to: + return None + + return self + + + def load_module(self, mod_fullname): + parent_frame = inspect.currentframe().f_back + + while in_importlib(parent_frame): + parent_frame = parent_frame.f_back + + module = fetch_module_from_frame(mod_fullname, parent_frame) + + # we HAVE to include the module in sys.modules, per the import PEP. + # older verions of python were more lenient about this being set, but + # not in >= python3.3, unfortunately. this requirement necessitates the + # ugly code in SelfWrapper.__call__ + sys.modules[mod_fullname] = module + module.__loader__ = self + + return module + + +def run_tests(env, locale, args, version, **extra_env): # pragma: no cover + py_version = "python" + py_version += str(version) + + py_bin = which(py_version) + return_code = None + + if py_bin: + print("Testing %s, locale %r" % (py_version.capitalize(), + locale)) + + env["LANG"] = locale + + for k,v in extra_env.items(): + env[k] = str(v) + + cmd = [py_bin, "-W", "ignore", os.path.join(THIS_DIR, "test.py")] + args[1:] + launch = lambda: os.spawnve(os.P_WAIT, cmd[0], cmd, env) + return_code = launch() + + return return_code + + + +# we're being run as a stand-alone script +if __name__ == "__main__": # pragma: no cover + def parse_args(): + from optparse import OptionParser + + parser = OptionParser() + parser.add_option("-e", "--envs", dest="envs", action="append") + parser.add_option("-l", "--locales", dest="constrain_locales", action="append") + options, args = parser.parse_args() + + envs = options.envs or [] + constrain_locales = options.constrain_locales or [] + + return args, envs, constrain_locales + + # these are essentially restrictions on what envs/constrain_locales to restrict to for + # the tests. if they're empty lists, it means use all available + args, constrain_versions, constrain_locales = parse_args() + action = None + if args: + action = args[0] + + if action in ("test", "travis"): + import test + coverage = None + if test.HAS_UNICODE_LITERAL: + import coverage + + env = os.environ.copy() + env["SH_TESTS_RUNNING"] = "1" + if coverage: + test.append_module_path(env, coverage) + + # if we're testing locally, run all versions of python on the system + if action == "test": + all_versions = ("2.6", "2.7", "3.1", "3.2", "3.3", "3.4", "3.5") + + # if we're testing on travis, just use the system's default python, + # since travis will spawn a vm per python version in our .travis.yml + # file + elif action == "travis": + v = sys.version_info + sys_ver = "%d.%d" % (v[0], v[1]) + all_versions = (sys_ver,) + + all_locales = ("en_US.UTF-8", "C") + i = 0 + for locale in all_locales: + if constrain_locales and locale not in constrain_locales: + continue + + for version in all_versions: + if constrain_versions and version not in constrain_versions: + continue + + env_copy = env.copy() + exit_code = run_tests(env_copy, locale, args, version, + SH_TEST_RUN_IDX=i) + + if exit_code is None: + print("Couldn't find %s, skipping" % version) + + elif exit_code != 0: + print("Failed for %s, %s" % (version, locale)) + exit(1) + + i += 1 + + ran_versions = ",".join(all_versions) + print("Tested Python versions: %s" % ran_versions) + + else: + env = Environment(globals()) + run_repl(env) + +# we're being imported from somewhere +else: + self = sys.modules[__name__] + sys.modules[__name__] = SelfWrapper(self) + register_importer() + From c1fcfd4d218f9ebf106ed6d9d292474d1fe6cdcc Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 27 Feb 2017 10:18:24 +0100 Subject: [PATCH 568/631] First Python integration test (#751) Reimplement the 'snabb lwaftr bench' test in Python --- .gitignore | 1 + src/program/lwaftr/bench/selftest.sh | 24 --------- src/program/lwaftr/tests/selftest.sh | 14 +++++ .../lwaftr/tests/subcommands/__init__.py | 0 .../lwaftr/tests/subcommands/branch_test.py | 54 +++++++++++++++++++ 5 files changed, 69 insertions(+), 24 deletions(-) delete mode 100755 src/program/lwaftr/bench/selftest.sh create mode 100755 src/program/lwaftr/tests/selftest.sh create mode 100644 src/program/lwaftr/tests/subcommands/__init__.py create mode 100644 src/program/lwaftr/tests/subcommands/branch_test.py diff --git a/.gitignore b/.gitignore index c3051f3928..6b9ab28531 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ obj *.so *.o *# +__pycache__ /src/snabbswitch /src/snabb /src/testlog diff --git a/src/program/lwaftr/bench/selftest.sh b/src/program/lwaftr/bench/selftest.sh deleted file mode 100755 index c728f005c6..0000000000 --- a/src/program/lwaftr/bench/selftest.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -TEST_DIR="./program/lwaftr/tests" - -source ${TEST_DIR}/common.sh - -check_for_root - -echo "Testing lwaftr bench" - -DATA_DIR="${TEST_DIR}/data" -BENCHDATA_DIR="${TEST_DIR}/benchdata" - -./snabb lwaftr bench --duration 1 --bench-file bench.csv \ - ${DATA_DIR}/icmp_on_fail.conf \ - ${BENCHDATA_DIR}/ipv{4,6}-0550.pcap &> /dev/null -assert_equal $? 0 "lwaftr bench failed with error code $?" -assert_file_exists ./bench.csv --remove - -./snabb lwaftr bench --reconfigurable --duration 1 --bench-file bench.csv \ - ${DATA_DIR}/icmp_on_fail.conf \ - ${BENCHDATA_DIR}/ipv{4,6}-0550.pcap &> /dev/null -assert_equal $? 0 "lwaftr bench --reconfigurable failed with error code $?" -assert_file_exists ./bench.csv --remove diff --git a/src/program/lwaftr/tests/selftest.sh b/src/program/lwaftr/tests/selftest.sh new file mode 100755 index 0000000000..23b4cd769b --- /dev/null +++ b/src/program/lwaftr/tests/selftest.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Make it work from wherever this script is called, and let tests know. +export TESTS_DIR=`dirname "$0"` + +# Entry point for Python tests. +# +# Start discovery from this script's directory, the root of the "tests" subtree. +# Find unittests in all Python files ending with "_test.py". +# List all executed tests, don't show just dots. +python3 -m unittest discover \ + --start-directory "${TESTS_DIR}" \ + --pattern "*_test.py" \ + --verbose diff --git a/src/program/lwaftr/tests/subcommands/__init__.py b/src/program/lwaftr/tests/subcommands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/program/lwaftr/tests/subcommands/branch_test.py b/src/program/lwaftr/tests/subcommands/branch_test.py new file mode 100644 index 0000000000..7862d62c62 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/branch_test.py @@ -0,0 +1,54 @@ +""" +Test the "snabb lwaftr bench" subcommand. Does not need NIC cards. +""" + +import os +from pathlib import Path +import unittest + +from lib import sh + + +# Commands run under "sudo" run as root. The root's user PATH should not +# include "." (the current directory) for security reasons. If this is the +# case, when we run tests from the "src" directory (where the "snabb" +# executable is), the "snabb" executable will not be found by relative paths. +# Therefore we make all paths absolute. +TESTS_DIR = Path(os.environ['TESTS_DIR']).resolve() +DATA_DIR = TESTS_DIR / 'data' +BENCHDATA_DIR = TESTS_DIR / 'benchdata' +SNABB_CMD = TESTS_DIR.parents[2] / 'snabb' +BENCHMARK_FILENAME = 'benchtest.csv' +# Snabb creates the benchmark file in the current directory +BENCHMARK_PATH = Path.cwd() / BENCHMARK_FILENAME + + +class TestBenchSubcommand(unittest.TestCase): + + cmd_args = ( + SNABB_CMD, 'lwaftr', 'bench', + '--duration', '0.1', + '--bench-file', BENCHMARK_FILENAME, + DATA_DIR / 'icmp_on_fail.conf', + BENCHDATA_DIR / 'ipv4-0550.pcap', + BENCHDATA_DIR / 'ipv6-0550.pcap', + ) + + def run_bench_test(self, cmd_args): + output = sh.sudo(*cmd_args) + self.assertEqual(output.exit_code, 0) + self.assertTrue(BENCHMARK_PATH.is_file(), + 'Cannot find {}'.format(BENCHMARK_PATH)) + BENCHMARK_PATH.unlink() + + def test_standard(self): + self.run_bench_test(self.cmd_args) + + def test_reconfigurable(self): + reconf_cmd_args = list(self.cmd_args) + reconf_cmd_args.insert(3, '--reconfigurable') + self.run_bench_test(reconf_cmd_args) + + +if __name__ == '__main__': + unittest.main() From da1acc65c8b12c2be4dde9544f8db89023480ca4 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 27 Feb 2017 15:44:31 +0100 Subject: [PATCH 569/631] Add 'run' subcommand test, refactor the 'bench' one (#753) --- src/program/lwaftr/tests/lib/test_env.py | 24 +++++++++++ .../lwaftr/tests/subcommands/branch_test.py | 30 ++++--------- .../lwaftr/tests/subcommands/run_test.py | 42 +++++++++++++++++++ 3 files changed, 74 insertions(+), 22 deletions(-) create mode 100644 src/program/lwaftr/tests/lib/test_env.py create mode 100644 src/program/lwaftr/tests/subcommands/run_test.py diff --git a/src/program/lwaftr/tests/lib/test_env.py b/src/program/lwaftr/tests/lib/test_env.py new file mode 100644 index 0000000000..8bebea5fa8 --- /dev/null +++ b/src/program/lwaftr/tests/lib/test_env.py @@ -0,0 +1,24 @@ +""" +Environment support code for tests. +""" + +import os +from pathlib import Path + + +# Commands run under "sudo" run as root. The root's user PATH should not +# include "." (the current directory) for security reasons. If this is the +# case, when we run tests from the "src" directory (where the "snabb" +# executable is), the "snabb" executable will not be found by relative paths. +# Therefore we make all paths absolute. +TESTS_DIR = Path(os.environ['TESTS_DIR']).resolve() +DATA_DIR = TESTS_DIR / 'data' +BENCHDATA_DIR = TESTS_DIR / 'benchdata' +SNABB_CMD = TESTS_DIR.parents[2] / 'snabb' +BENCHMARK_FILENAME = 'benchtest.csv' +# Snabb creates the benchmark file in the current directory +BENCHMARK_PATH = Path.cwd() / BENCHMARK_FILENAME + + +def nic_names(): + return os.environ.get('SNABB_PCI0'), os.environ.get('SNABB_PCI1') diff --git a/src/program/lwaftr/tests/subcommands/branch_test.py b/src/program/lwaftr/tests/subcommands/branch_test.py index 7862d62c62..90ff6aa6fc 100644 --- a/src/program/lwaftr/tests/subcommands/branch_test.py +++ b/src/program/lwaftr/tests/subcommands/branch_test.py @@ -2,28 +2,14 @@ Test the "snabb lwaftr bench" subcommand. Does not need NIC cards. """ -import os -from pathlib import Path import unittest from lib import sh +from lib.test_env import ( + BENCHMARK_FILENAME, BENCHMARK_PATH, DATA_DIR, BENCHDATA_DIR, SNABB_CMD) -# Commands run under "sudo" run as root. The root's user PATH should not -# include "." (the current directory) for security reasons. If this is the -# case, when we run tests from the "src" directory (where the "snabb" -# executable is), the "snabb" executable will not be found by relative paths. -# Therefore we make all paths absolute. -TESTS_DIR = Path(os.environ['TESTS_DIR']).resolve() -DATA_DIR = TESTS_DIR / 'data' -BENCHDATA_DIR = TESTS_DIR / 'benchdata' -SNABB_CMD = TESTS_DIR.parents[2] / 'snabb' -BENCHMARK_FILENAME = 'benchtest.csv' -# Snabb creates the benchmark file in the current directory -BENCHMARK_PATH = Path.cwd() / BENCHMARK_FILENAME - - -class TestBenchSubcommand(unittest.TestCase): +class TestBench(unittest.TestCase): cmd_args = ( SNABB_CMD, 'lwaftr', 'bench', @@ -34,20 +20,20 @@ class TestBenchSubcommand(unittest.TestCase): BENCHDATA_DIR / 'ipv6-0550.pcap', ) - def run_bench_test(self, cmd_args): + def execute_bench_test(self, cmd_args): output = sh.sudo(*cmd_args) self.assertEqual(output.exit_code, 0) self.assertTrue(BENCHMARK_PATH.is_file(), 'Cannot find {}'.format(BENCHMARK_PATH)) BENCHMARK_PATH.unlink() - def test_standard(self): - self.run_bench_test(self.cmd_args) + def test_bench_standard(self): + self.execute_bench_test(self.cmd_args) - def test_reconfigurable(self): + def test_bench_reconfigurable(self): reconf_cmd_args = list(self.cmd_args) reconf_cmd_args.insert(3, '--reconfigurable') - self.run_bench_test(reconf_cmd_args) + self.execute_bench_test(reconf_cmd_args) if __name__ == '__main__': diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py new file mode 100644 index 0000000000..5ea6a06586 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -0,0 +1,42 @@ +""" +Test the "snabb lwaftr run" subcommand. Needs NIC names. +""" + +import unittest + +from lib import sh +from lib.test_env import DATA_DIR, SNABB_CMD, nic_names + + +SNABB_PCI0, SNABB_PCI1 = nic_names() + + +class TestRun(unittest.TestCase): + + cmd_args = ( + SNABB_CMD, 'lwaftr', 'run', + '--duration', '0.1', + '--bench-file', '/dev/null', + '--conf', DATA_DIR / 'icmp_on_fail.conf', + '--v4', SNABB_PCI0, + '--v6', SNABB_PCI1, + ) + + def execute_run_test(self, cmd_args): + output = sh.sudo(*cmd_args) + self.assertEqual(output.exit_code, 0) + self.assert_(len(output.splitlines()) > 1) + + @unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') + def test_run_standard(self): + self.execute_run_test(self.cmd_args) + + @unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') + def test_run_reconfigurable(self): + reconf_cmd_args = list(self.cmd_args) + reconf_cmd_args.insert(3, '--reconfigurable') + self.execute_run_test(reconf_cmd_args) + + +if __name__ == '__main__': + unittest.main() From a80f4815381ecd384b9c5992973b76c33740a14e Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 28 Feb 2017 12:01:57 +0100 Subject: [PATCH 570/631] #748 - Re-enabled broken tests --- src/program/lwaftr/query/tests/test_query.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/program/lwaftr/query/tests/test_query.sh b/src/program/lwaftr/query/tests/test_query.sh index b6bfd3bfcb..cc816f6bc2 100755 --- a/src/program/lwaftr/query/tests/test_query.sh +++ b/src/program/lwaftr/query/tests/test_query.sh @@ -35,9 +35,8 @@ if [[ -n "$pid" ]]; then fi # Test query by name. -## FIXME: currently broken in non-reconfigurable mode. -#test_lwaftr_query "--name $LWAFTR_NAME" -#test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" -#test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" +test_lwaftr_query "--name $LWAFTR_NAME" +test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" +test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" exit 0 From 8589b56800d98e9d6e8ccceaa3b7f9e086e11593 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 28 Feb 2017 13:14:09 +0100 Subject: [PATCH 571/631] Fix #748 - Move claim_name to lwaftr_app This fixes the regression reported in #748. The regression would mean that the lwAFTR didn't specify a name when one was given without the reconfigurable flag enabled. The fix was to move the call to `claim_name` to `lwaftr_app` which is called both when `--reconfigurable` is given and without it. --- src/program/lwaftr/setup.lua | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index b038777b3c..074d75a686 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -35,6 +35,24 @@ function lwaftr_app(c, conf) local function append(t, elem) table.insert(t, elem) end local function prepend(t, elem) table.insert(t, 1, elem) end + -- Claim the name if one is defined. + local function switch_names(config) + local currentname = engine.program_name + local name = config.softwire_config.name + -- Don't do anything if the name isn't set. + if name == nil then + return + end + + local success, err = pcall(engine.claim_name, name) + if success == false then + -- Restore the previous name. + config.softwire_config.name = currentname + assert(success, err) + end + end + switch_names(conf) + config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = external_interface.reassembly.max_packets, @@ -524,26 +542,10 @@ end function reconfigurable(scheduling, f, graph, conf, ...) local args = {...} - local function switch_names(conf) - local currentname = engine.program_name - local name = conf.apps.lwaftr.arg.softwire_config.name - -- Don't do anything if the name isn't set. - if name == nil then - return - end - - local success, err = pcall(engine.claim_name, name) - if success == false then - -- Restore the previous name. - conf.apps.lwaftr.arg.softwire_config.name = currentname - assert(success, err) - end - end local function setup_fn(conf) local graph = config.new() f(graph, conf, unpack(args)) - switch_names(graph) return graph end From 73abb702f826d697db8fc6757405767a1732993e Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 28 Feb 2017 14:04:16 +0100 Subject: [PATCH 572/631] Fix indentation nit --- src/program/lwaftr/setup.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 074d75a686..ca2c7cea16 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -35,8 +35,8 @@ function lwaftr_app(c, conf) local function append(t, elem) table.insert(t, elem) end local function prepend(t, elem) table.insert(t, 1, elem) end - -- Claim the name if one is defined. - local function switch_names(config) + -- Claim the name if one is defined. + local function switch_names(config) local currentname = engine.program_name local name = config.softwire_config.name -- Don't do anything if the name isn't set. From 40a980b0b2702e3fbc72ec02ce1cba529a909b70 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Tue, 28 Feb 2017 16:37:23 +0100 Subject: [PATCH 573/631] Add a selftest for the "lwaftr loadtest" command (#755) --- .../{branch_test.py => bench_test.py} | 0 .../lwaftr/tests/subcommands/loadtest_test.py | 56 +++++++++++++++++++ .../lwaftr/tests/subcommands/run_test.py | 5 +- 3 files changed, 58 insertions(+), 3 deletions(-) rename src/program/lwaftr/tests/subcommands/{branch_test.py => bench_test.py} (100%) create mode 100644 src/program/lwaftr/tests/subcommands/loadtest_test.py diff --git a/src/program/lwaftr/tests/subcommands/branch_test.py b/src/program/lwaftr/tests/subcommands/bench_test.py similarity index 100% rename from src/program/lwaftr/tests/subcommands/branch_test.py rename to src/program/lwaftr/tests/subcommands/bench_test.py diff --git a/src/program/lwaftr/tests/subcommands/loadtest_test.py b/src/program/lwaftr/tests/subcommands/loadtest_test.py new file mode 100644 index 0000000000..08ae622f2e --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/loadtest_test.py @@ -0,0 +1,56 @@ +""" +Test the "snabb lwaftr loadtest" subcommand. Needs NIC names. + +Since there are only two NIC names available in snabb-bot, and we need to +execute two programs networked to each other ("run" and "loadtest"), they +are set to on-a-stick mode, so that they use one NIC each instead of two. +""" + +import unittest + +from lib import sh +from lib.test_env import BENCHDATA_DIR, DATA_DIR, SNABB_CMD, nic_names + + +SNABB_PCI0, SNABB_PCI1 = nic_names() + + +@unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') +class TestLoadtest(unittest.TestCase): + + run_cmd_args = ( + SNABB_CMD, 'lwaftr', 'run', + '--bench-file', '/dev/null', + '--conf', DATA_DIR / 'icmp_on_fail.conf', + '--on-a-stick', SNABB_PCI0, + ) + + loadtest_cmd_args = ( + SNABB_CMD, 'lwaftr', 'loadtest', + '--bench-file', '/dev/null', + # Something quick and easy. + '--program', 'ramp_up', + '--step', '0.1e8', + '--duration', '0.1', + '--bitrate', '0.2e8', + # Just one card for on-a-stick mode. + BENCHDATA_DIR / 'ipv4_and_ipv6_stick_imix.pcap', 'ALL', 'ALL', SNABB_PCI1, + ) + + # Use setUpClass to only setup the "run" daemon once for all tests. + @classmethod + def setUpClass(cls): + cls.run_cmd = sh.sudo(*cls.run_cmd_args, _bg=True) + + def test_loadtest(self): + output = sh.sudo(*self.loadtest_cmd_args) + self.assertEqual(output.exit_code, 0) + self.assertTrue(len(output.splitlines()) > 10) + + @classmethod + def tearDownClass(cls): + cls.run_cmd.terminate() + + +if __name__ == '__main__': + unittest.main() diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py index 5ea6a06586..a697dab8bc 100644 --- a/src/program/lwaftr/tests/subcommands/run_test.py +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -11,6 +11,7 @@ SNABB_PCI0, SNABB_PCI1 = nic_names() +@unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') class TestRun(unittest.TestCase): cmd_args = ( @@ -25,13 +26,11 @@ class TestRun(unittest.TestCase): def execute_run_test(self, cmd_args): output = sh.sudo(*cmd_args) self.assertEqual(output.exit_code, 0) - self.assert_(len(output.splitlines()) > 1) + self.assertTrue(len(output.splitlines()) > 1) - @unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') def test_run_standard(self): self.execute_run_test(self.cmd_args) - @unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') def test_run_reconfigurable(self): reconf_cmd_args = list(self.cmd_args) reconf_cmd_args.insert(3, '--reconfigurable') From 4c92a0ec0788ba5f675513f9cb347f1b3d564594 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 28 Feb 2017 17:28:01 +0100 Subject: [PATCH 574/631] Fix #702 - Error when neither of next-hop's leaves are specified The 'next-hop' container contains two leaves 'ip' and 'mac'. One of the two leaves must be specified or both must be specified. There isn't a good method to constrain this in YANG so an error occurs in the lwAFTR if neither are specified. --- src/program/lwaftr/setup.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index ca2c7cea16..0b3cf916f1 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -53,6 +53,11 @@ function lwaftr_app(c, conf) end switch_names(conf) + -- Verify either or both of next-hop value's are specified (can't be done in YANG) + if internal_interface.next_hop == nil then + error("One or both of the 'next_hop' values must be specified") + end + config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = external_interface.reassembly.max_packets, From 4ea06f2318b766447d544d7fad710f2c585ccf9f Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Wed, 1 Mar 2017 14:00:05 +0100 Subject: [PATCH 575/631] Add test for 'lwaftr check' subcommand (#760) --- .../lwaftr/tests/data/counters/empty.lua | 4 +++ src/program/lwaftr/tests/lib/test_env.py | 1 + .../lwaftr/tests/subcommands/check_test.py | 35 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 src/program/lwaftr/tests/data/counters/empty.lua create mode 100644 src/program/lwaftr/tests/subcommands/check_test.py diff --git a/src/program/lwaftr/tests/data/counters/empty.lua b/src/program/lwaftr/tests/data/counters/empty.lua new file mode 100644 index 0000000000..e06737beed --- /dev/null +++ b/src/program/lwaftr/tests/data/counters/empty.lua @@ -0,0 +1,4 @@ +return { + ["memuse-ipv4-frag-reassembly-buffer"] = 463571780, + ["memuse-ipv6-frag-reassembly-buffer"] = 464727376, +} diff --git a/src/program/lwaftr/tests/lib/test_env.py b/src/program/lwaftr/tests/lib/test_env.py index 8bebea5fa8..6c6f4b14ea 100644 --- a/src/program/lwaftr/tests/lib/test_env.py +++ b/src/program/lwaftr/tests/lib/test_env.py @@ -13,6 +13,7 @@ # Therefore we make all paths absolute. TESTS_DIR = Path(os.environ['TESTS_DIR']).resolve() DATA_DIR = TESTS_DIR / 'data' +COUNTERS_DIR = DATA_DIR / 'counters' BENCHDATA_DIR = TESTS_DIR / 'benchdata' SNABB_CMD = TESTS_DIR.parents[2] / 'snabb' BENCHMARK_FILENAME = 'benchtest.csv' diff --git a/src/program/lwaftr/tests/subcommands/check_test.py b/src/program/lwaftr/tests/subcommands/check_test.py new file mode 100644 index 0000000000..07f5c5dbf4 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/check_test.py @@ -0,0 +1,35 @@ +""" +Test the "snabb lwaftr check" subcommand. Does not need NIC names. +""" + +import unittest + +from lib import sh +from lib.test_env import COUNTERS_DIR, DATA_DIR, SNABB_CMD + + +class TestCheck(unittest.TestCase): + + cmd_args = ( + SNABB_CMD, 'lwaftr', 'check', + DATA_DIR / 'icmp_on_fail.conf', + DATA_DIR / 'empty.pcap', DATA_DIR / 'empty.pcap', + '/dev/null', '/dev/null', + COUNTERS_DIR / 'empty.lua', + ) + + def execute_check_test(self, cmd_args): + output = sh.sudo(*cmd_args) + self.assertEqual(output.exit_code, 0) + + def test_check_standard(self): + self.execute_check_test(self.cmd_args) + + def test_check_on_a_stick(self): + onastick_cmd_args = list(self.cmd_args) + onastick_cmd_args.insert(3, '--on-a-stick') + self.execute_check_test(onastick_cmd_args) + + +if __name__ == '__main__': + unittest.main() From 9f18b41c21557a6b0ca9fd12f3ced195bd973d04 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Thu, 2 Mar 2017 13:34:02 +0100 Subject: [PATCH 576/631] Add a selftest for the "lwaftr monitor" command (#762) --- src/program/lwaftr/tests/lib/test_env.py | 14 +++++ .../lwaftr/tests/subcommands/monitor_test.py | 54 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 src/program/lwaftr/tests/subcommands/monitor_test.py diff --git a/src/program/lwaftr/tests/lib/test_env.py b/src/program/lwaftr/tests/lib/test_env.py index 6c6f4b14ea..8cbda5708e 100644 --- a/src/program/lwaftr/tests/lib/test_env.py +++ b/src/program/lwaftr/tests/lib/test_env.py @@ -5,6 +5,8 @@ import os from pathlib import Path +from lib import sh + # Commands run under "sudo" run as root. The root's user PATH should not # include "." (the current directory) for security reasons. If this is the @@ -23,3 +25,15 @@ def nic_names(): return os.environ.get('SNABB_PCI0'), os.environ.get('SNABB_PCI1') + + +def tap_name(): + """ + Return the first TAP interface name if one found: (tap_iface, None). + Return (None, 'No TAP interface available') if none found. + """ + output = sh.ip('tuntap', 'list') + tap_iface = output.split(':')[0] + if not tap_iface: + return None, 'No TAP interface available' + return tap_iface, None diff --git a/src/program/lwaftr/tests/subcommands/monitor_test.py b/src/program/lwaftr/tests/subcommands/monitor_test.py new file mode 100644 index 0000000000..35b8972ed8 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/monitor_test.py @@ -0,0 +1,54 @@ +""" +Test the "snabb lwaftr monitor" subcommand. Needs a NIC name and a TAP interface. + +1. Execute "snabb lwaftr run" in on-a-stick mode and with the mirror option set. +2. Run "snabb lwaftr monitor" to set the counter and check its output. +""" + +import unittest + +from lib import sh +from lib.test_env import DATA_DIR, SNABB_CMD, nic_names, tap_name + + +SNABB_PCI0 = nic_names()[0] +TAP_IFACE, tap_err_msg = tap_name() + + +@unittest.skipUnless(SNABB_PCI0, 'NIC not configured') +@unittest.skipUnless(TAP_IFACE, tap_err_msg) +class TestMonitor(unittest.TestCase): + + run_cmd_args = ( + SNABB_CMD, 'lwaftr', 'run', + '--name', 'monitor_test', + '--bench-file', '/dev/null', + '--conf', DATA_DIR / 'icmp_on_fail.conf', + '--on-a-stick', SNABB_PCI0, + '--mirror', TAP_IFACE, + ) + + monitor_cmd_args = ( + SNABB_CMD, 'lwaftr', 'monitor', + '--name', 'monitor_test', + 'all', + ) + + # Use setUpClass to only setup the "run" daemon once for all tests. + @classmethod + def setUpClass(cls): + cls.run_cmd = sh.sudo(*cls.run_cmd_args, _bg=True) + + def test_monitor(self): + output = sh.sudo(*self.monitor_cmd_args) + self.assertEqual(output.exit_code, 0) + self.assertIn('Mirror address set', output) + self.assertIn('255.255.255.255', output) + + @classmethod + def tearDownClass(cls): + cls.run_cmd.terminate() + + +if __name__ == '__main__': + unittest.main() From fcaa19f186c4b4d41207c12d5955348531b18745 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 3 Mar 2017 21:07:40 +0100 Subject: [PATCH 577/631] Add test for #727 - check softwires are matched This adds tests to check that softwires that have no PSID map counterpart cause an error to be raised. It tests that this occurs when adding or setting one via snabb config and when running lwaftr bench and lwaftr run. --- .../tests/data/missing_softwire_psidmap.conf | 50 +++++++++++++++++++ .../lwaftr/tests/subcommands/bench_test.py | 10 ++++ .../lwaftr/tests/subcommands/config_test.py | 14 ++++++ .../lwaftr/tests/subcommands/run_test.py | 11 ++++ 4 files changed, 85 insertions(+) create mode 100644 src/program/lwaftr/tests/data/missing_softwire_psidmap.conf diff --git a/src/program/lwaftr/tests/data/missing_softwire_psidmap.conf b/src/program/lwaftr/tests/data/missing_softwire_psidmap.conf new file mode 100644 index 0000000000..c07502cf06 --- /dev/null +++ b/src/program/lwaftr/tests/data/missing_softwire_psidmap.conf @@ -0,0 +1,50 @@ +softwire-config { + binding-table { + br-address 8:9:a:b:c:d:e:f; + br-address 1e:1:1:1:1:1:1:af; + br-address 1e:2:2:2:2:2:2:af; + psid-map { + addr 178.79.150.15; + psid-length 4; + shift 12; + } + softwire { + ipv4 178.79.150.233; + psid 4660; + b4-ipv6 127:11:12:13:14:15:16:128; + } + softwire { + ipv4 178.79.150.9; + psid 72; + b4-ipv6 127:12:13:14:15:16:17:128; + } + } + external-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 10.10.10.10; + mac 12:12:12:12:12:12; + next-hop { + mac 68:68:68:68:68:68; + } + reassembly { + max-fragments-per-packet 40; + } + } + internal-interface { + allow-incoming-icmp false; + error-rate-limiting { + packets 600000; + } + ip 8:9:a:b:c:d:e:f; + mac 22:22:22:22:22:22; + next-hop { + mac 44:44:44:44:44:44; + } + reassembly { + max-fragments-per-packet 40; + } + } +} diff --git a/src/program/lwaftr/tests/subcommands/bench_test.py b/src/program/lwaftr/tests/subcommands/bench_test.py index 8b1056a5a2..33b4faa18f 100644 --- a/src/program/lwaftr/tests/subcommands/bench_test.py +++ b/src/program/lwaftr/tests/subcommands/bench_test.py @@ -33,6 +33,16 @@ def test_bench_reconfigurable(self): reconf_args.insert(3, '--reconfigurable') self.execute_bench_test(reconf_args) + def test_config_with_invalid_softwire(self): + config_file = str(DATA_DIR / "missing_softwire_psidmap.conf") + invalid_softwire_args = list(self.cmd_args) + invalid_softwire_args[-3] = config_file + # Verify it errors when there is a softwire lacking a PSID mapping entry + self.assertRaises( + AssertionError, + self.execute_bench_test, + invalid_softwire_args + ) if __name__ == '__main__': unittest.main() diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py index 393ec8e7d9..5a8c6d0c8f 100644 --- a/src/program/lwaftr/tests/subcommands/config_test.py +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -251,6 +251,20 @@ def test_set(self): self.assertEqual(output.strip(), bytes(test_psid, ENC), '\n'.join(('OUTPUT', str(output, ENC)))) + def test_softwire_not_in_psidmap(self): + """ + Tests that softwire with a PSID out of PSID map errors + """ + # Create a softwire which won't have an IPv4 address in the PSID map. + test_softwire = "{ ipv4 192.168.1.23; psid 72; b4-ipv6 ::1; } " + + add_args = self.get_cmd_args('add') + add_args.extend(( + "/softwire-config/binding-table/softwire", + test_softwire, + )) + self.assertRaises(AssertionError, self.run_cmd, add_args) + if __name__ == '__main__': unittest.main() diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py index a48dd0d3a5..ba6adb8479 100644 --- a/src/program/lwaftr/tests/subcommands/run_test.py +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -35,6 +35,17 @@ def test_run_reconfigurable(self): reconf_args.insert(3, '--reconfigurable') self.execute_run_test(reconf_args) + def test_config_with_invalid_softwire(self): + config_file = str(DATA_DIR / "missing_softwire_psidmap.conf") + invalid_softwire_args = list(self.cmd_args) + invalid_softwire_args[-5] = config_file + # Verify it errors when there is a softwire lacking a PSID mapping entry + self.assertRaises( + AssertionError, + self.execute_run_test, + invalid_softwire_args + ) + if __name__ == '__main__': unittest.main() From 7236a672c005b1998f8908b2deb6c5b5abb697c1 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 3 Mar 2017 21:13:00 +0100 Subject: [PATCH 578/631] Add validation for softwires using snabb config add This adds checking for adding softwires without a PSID map counterpart that will now produce an error when it is attempted. This is to partly address issue #727. --- src/apps/config/support/snabb-softwire-v1.lua | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index cd86d5781a..ff7b750a66 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -2,21 +2,39 @@ module(..., package.seeall) local ffi = require('ffi') local app = require('core.app') +local corelib = require('core.lib') local equal = require('core.lib').equal local dirname = require('core.lib').dirname local data = require('lib.yang.data') local ipv4_ntop = require('lib.yang.util').ipv4_ntop +local ipv4 = require('lib.protocol.ipv4') local ipv6 = require('lib.protocol.ipv6') local yang = require('lib.yang.yang') local ctable = require('lib.ctable') local cltable = require('lib.cltable') local path_mod = require('lib.yang.path') +local util = require("lib.yang.util") local generic = require('apps.config.support').generic_schema_config_support +-- Validates that the softwire is in the PSID mapping, returns true if the +-- softwire is valid (i.e. has a corresponding PSID mapping), false if invalid. +local function validate_softwire_psid_mapping(graph, softwire) + local function convert_ipv4_to_key(val) + local key = corelib.htonl(val) -- Convert endianness + return ipv4:pton(ipv4_ntop(key)) + end + local bt = graph.apps.lwaftr.arg.softwire_config.binding_table + local psidmap_key = convert_ipv4_to_key(softwire.key.ipv4) + local psidmap_entry = cltable.get(bt.psid_map, psidmap_key) + local ip = ipv4_ntop(softwire.key.ipv4) + assert(psidmap_entry, "No PSID map for softwire '"..ip.."'") +end + local function add_softwire_entry_actions(app_graph, entries) assert(app_graph.apps['lwaftr']) local ret = {} for entry in entries:iterate() do + validate_softwire_psid_mapping(app_graph, entry) local blob = entries.entry_type() ffi.copy(blob, entry, ffi.sizeof(blob)) local args = {'lwaftr', 'add_softwire_entry', blob} From 04d96054157865aa4271116266033d278ba34bb1 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Sat, 4 Mar 2017 10:20:40 +0100 Subject: [PATCH 579/631] Fix pervious config test with new validation Previously we had a test which added a softwire and read it back to check that softwires can be added and read back correctly. This added a softwire which didn't have a PSID map entry. This test fixes that. --- src/program/lwaftr/tests/subcommands/config_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py index 5a8c6d0c8f..103cacd2ea 100644 --- a/src/program/lwaftr/tests/subcommands/config_test.py +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -137,6 +137,14 @@ def test_add(self): """ Add a softwire section, get it back and check all the values. """ + # Add a PSID map for the IP we're going to use. + psidmap_add_args = self.get_cmd_args('add') + psidmap_add_args.extend(( + '/softwire-config/binding-table/psid-map', + '{ addr 1.2.3.4; psid-length 16; }' + )) + self.run_cmd(psidmap_add_args) + # External IPv4. add_args = self.get_cmd_args('add') add_args.extend(( From 8e6b3988903078765494def511cb9009dfafc97a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Sat, 4 Mar 2017 13:40:08 +0100 Subject: [PATCH 580/631] Improve tests for softwire validation Adds error messages to explain why tests have failed so it's easier to see what the problem is. I have also extended the config add test to verify it not only errors but also doesn't add the softwire. --- src/program/lwaftr/tests/subcommands/bench_test.py | 8 +++----- src/program/lwaftr/tests/subcommands/config_test.py | 12 +++++++++++- src/program/lwaftr/tests/subcommands/run_test.py | 9 +++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/program/lwaftr/tests/subcommands/bench_test.py b/src/program/lwaftr/tests/subcommands/bench_test.py index 33b4faa18f..369e9cf663 100644 --- a/src/program/lwaftr/tests/subcommands/bench_test.py +++ b/src/program/lwaftr/tests/subcommands/bench_test.py @@ -38,11 +38,9 @@ def test_config_with_invalid_softwire(self): invalid_softwire_args = list(self.cmd_args) invalid_softwire_args[-3] = config_file # Verify it errors when there is a softwire lacking a PSID mapping entry - self.assertRaises( - AssertionError, - self.execute_bench_test, - invalid_softwire_args - ) + err = "Started with config file that has softwire without PSID mapping" + with self.assertRaises(AssertionError, msg=err): + self.execute_bench_test(invalid_softwire_args) if __name__ == '__main__': unittest.main() diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py index 103cacd2ea..78ed469289 100644 --- a/src/program/lwaftr/tests/subcommands/config_test.py +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -271,8 +271,18 @@ def test_softwire_not_in_psidmap(self): "/softwire-config/binding-table/softwire", test_softwire, )) - self.assertRaises(AssertionError, self.run_cmd, add_args) + add_error = "Able to add softwire with without PSID mapping" + with self.assertRaises(AssertionError, msg=add_error): + self.run_cmd(add_args) + # Then try and get the softwire added to ensure it's not been added + get_args = self.get_cmd_args('get') + get_args.append( + "/softwire-config/binding-table/softwire[ipv4=192.168.1.23][psid=72]" + ) + get_error = "Softwire was added with an invalid PSID mapping." + with self.assertRaises(AssertionError, msg=get_error): + self.run_cmd(get_args) if __name__ == '__main__': unittest.main() diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py index ba6adb8479..7a3ab23bdb 100644 --- a/src/program/lwaftr/tests/subcommands/run_test.py +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -40,12 +40,9 @@ def test_config_with_invalid_softwire(self): invalid_softwire_args = list(self.cmd_args) invalid_softwire_args[-5] = config_file # Verify it errors when there is a softwire lacking a PSID mapping entry - self.assertRaises( - AssertionError, - self.execute_run_test, - invalid_softwire_args - ) - + err = "Started with config file that has softwire without PSID mapping" + with self.assertRaises(AssertionError, msg=err): + self.execute_run_test(invalid_softwire_args) if __name__ == '__main__': unittest.main() From 7733f20d623ddf39f3cd3fd91ccee145698407b8 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 6 Mar 2017 16:34:39 +0100 Subject: [PATCH 581/631] Fix #638 - Validate path on snabb config get --- src/program/config/common.lua | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/program/config/common.lua b/src/program/config/common.lua index 834e7755ab..6b6dcaa3d5 100644 --- a/src/program/config/common.lua +++ b/src/program/config/common.lua @@ -35,6 +35,19 @@ function data_parser(schema_name, path) return data.data_parser_from_grammar(path_grammar(schema_name, path)) end +function error_and_quit(err) + io.stderr:write(err .. "\n") + io.stderr:flush() + os.exit(1) +end + +function validate_path(schema_name, path) + local succ, err = pcall(path_grammar, schema_name, path) + if succ == false then + error_and_quit(err) + end +end + function parse_command_line(args, opts) opts = lib.parse(opts, parse_command_line_opts) local function err(msg) show_usage(opts.command, 1, msg) end @@ -65,8 +78,8 @@ function parse_command_line(args, opts) end if opts.with_path then if #args == 0 then err("missing path argument") end - -- FIXME: Validate path? ret.path = table.remove(args, 1) + validate_path(ret.schema_name, ret.path) end if opts.with_value then local parser = data_parser(ret.schema_name, ret.path) From df96eacd62f796f7b3212070a0ef11f596b211c4 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sun, 8 Jan 2017 23:10:16 +0100 Subject: [PATCH 582/631] Refactor selftest --- src/apps/lwaftr/nh_fwd.lua | 103 +++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index de50908b4b..9d351a650c 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -71,9 +71,6 @@ end local function get_ipv4_src_address(ptr) return rd32(get_ipv4_src_ptr(ptr)) end -local function get_ipv4_checksum_ptr (ptr) - return ptr + o_ipv4_checksum -end local function get_ipv6_next_header(ptr) return ptr[o_ipv6_next_header] end @@ -89,16 +86,6 @@ end local function copy_ether(dst, src) ffi.copy(dst, src, 6) end -local function copy_ipv4(dst, src) - ffi.copy(dst, src, 4) -end -local function copy_ipv6(dst, src) - ffi.copy(dst, src, 16) -end -local function get_ipv4_header_length(ptr) - local ver_and_ihl = ptr[0] - return lshift(band(ver_and_ihl, 0xf), 2) -end -- Set a bogus source IP address fe80::, so we can recognize it later when -- it comes back from the VM. @@ -112,15 +99,13 @@ end -- is a better way. -- local function ipv6_cache_trigger (pkt, mac) - local ether_dhost = get_ether_dhost_ptr(pkt) - local ipv6_hdr = get_ethernet_payload(pkt) - local ipv6_src_ip = get_ipv6_src_address(ipv6_hdr) + local ether_hdr = ethernet:new_from_mem(pkt.data, ethernet_header_size) + local ip_hdr = ipv6:new_from_mem(pkt.data + ethernet_header_size, pkt.length - ethernet_header_size) -- VM will discard packets not matching its MAC address on the interface. - copy_ether(ether_dhost, mac) - + ether_hdr:dst(mac) -- Set a bogus source IP address. - copy_ipv6(ipv6_src_ip, n_cache_src_ipv6) + ip_hdr:src(n_cache_src_ipv6) return pkt end @@ -130,21 +115,15 @@ local function send_ipv6_cache_trigger (r, pkt, mac) end local function ipv4_cache_trigger (pkt, mac) - local ether_dhost = get_ether_dhost_ptr(pkt) - local ipv4_hdr = get_ethernet_payload(pkt) - local ipv4_hdr_size = get_ipv4_header_length(ipv4_hdr) - local ipv4_src_ip = get_ipv4_src_ptr(ipv4_hdr) - local ipv4_checksum = get_ipv4_checksum_ptr(ipv4_hdr) + local ether_hdr = ethernet:new_from_mem(pkt.data, ethernet_header_size) + local ip_hdr = ipv4:new_from_mem(pkt.data + ethernet_header_size, pkt.length - ethernet_header_size) -- VM will discard packets not matching its MAC address on the interface. - copy_ether(ether_dhost, mac) - + ether_hdr:dst(mac) -- Set a bogus source IP address. - copy_ipv4(ipv4_src_ip, n_cache_src_ipv4) - - -- Clear checksum to recalculate it with new source IPv4 address. - wr16(ipv4_checksum, 0) - wr16(ipv4_checksum, htons(ipsum(pkt.data + ethernet_header_size, ipv4_hdr_size, 0))) + ip_hdr:src(n_cache_src_ipv4) + -- Recalculate checksum. + ip_hdr:checksum() return pkt end @@ -582,22 +561,46 @@ local function test_ipv6_flow () test_ipv6_service_to_vm({pkt1}) end -local function test_ipv4_cache_trigger () - local pkt = packet.from_string(lib.hexundump([[ +local function test_ipv4_cache_trigger (pkt) + local ether_dhost = "52:54:00:00:00:01" + local refresh_packet = ipv4_cache_trigger(pkt, ethernet:pton(ether_dhost)) + local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) + local ip_hdr = ipv4:new_from_mem(refresh_packet.data + ethernet_header_size, pkt.length - ethernet_header_size) + assert(ip_hdr:src_eq(n_cache_src_ipv4)) + assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) +end + +local function test_ipv6_cache_trigger (pkt) + local ether_dhost = "52:54:00:00:00:01" + local refresh_packet = ipv6_cache_trigger(pkt, ethernet:pton(ether_dhost)) + local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) + local ip_hdr = ipv6:new_from_mem(refresh_packet.data + ethernet_header_size, pkt.length - ethernet_header_size) + assert(ip_hdr:src_eq(n_cache_src_ipv6)) + assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) +end + +local function ipv4_udp_pkt () + return packet.from_string(lib.hexundump([[ 02:aa:aa:aa:aa:aa 02:99:99:99:99:99 08 00 45 00 02 18 00 00 00 00 0f 11 d3 61 0a 0a 0a 01 c1 05 01 64 30 39 04 00 00 26 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ]], 72)) - local ether_dhost = "52:54:00:00:00:01" - local refresh_packet = ipv4_cache_trigger(pkt, ethernet:pton(ether_dhost)) - local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) - assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) end -local function test_ipv6_cache_trigger () - local pkt = packet.from_string(lib.hexundump([[ +local function ipv4_tcp_pkt () + return packet.from_string(lib.hexundump([[ + 18 55 0f ae d0 1d a0 88 b4 2c fa ac 08 00 45 00 + 00 34 b8 a0 40 00 40 06 69 55 c0 a8 00 11 97 65 + c0 af e5 97 00 50 15 91 83 d6 5d 31 61 91 80 10 + 02 a9 ff 98 00 00 01 01 08 0a 07 c3 3c f9 47 c4 + 91 1a + ]], 66)) +end + +local function ipv6_udp_pkt () + return packet.from_string(lib.hexundump([[ 02:aa:aa:aa:aa:aa 02:99:99:99:99:99 86 dd 60 00 01 f0 01 f0 04 ff fc 00 00 01 00 02 00 03 00 04 00 05 00 00 00 7e fc 00 00 00 00 00 00 00 00 00 @@ -605,16 +608,26 @@ local function test_ipv6_cache_trigger () d3 89 c1 05 01 64 0a 0a 0a 01 04 00 30 39 00 0c 00 00 00 00 00 00 ]], 86)) - local ether_dhost = "52:54:00:00:00:01" - local refresh_packet = ipv6_cache_trigger(pkt, ethernet:pton(ether_dhost)) - local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) - assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) +end + +local function ipv6_tcp_pkt () + return packet.from_string(lib.hexundump([[ + 02 aa aa aa aa aa 02 99 99 99 99 99 86 dd 60 00 + 01 f0 01 f0 04 ff fc 00 00 01 00 02 00 03 00 04 + 00 05 00 00 00 7e fc 00 00 00 00 00 00 00 00 00 + 00 00 00 00 01 00 45 00 00 34 b8 a0 40 00 40 06 + 69 55 c0 a8 00 11 97 65 c0 af e5 97 00 50 15 91 + 83 d6 5d 31 61 91 80 10 02 a9 ff 98 00 00 01 01 + 08 0a 07 c3 3c f9 47 c4 91 1a + ]], 106)) end function selftest () print("nh_fwd: selftest") test_ipv4_flow() test_ipv6_flow() - test_ipv4_cache_trigger() - test_ipv6_cache_trigger() + test_ipv4_cache_trigger(ipv4_udp_pkt()) + test_ipv4_cache_trigger(ipv4_tcp_pkt()) + test_ipv6_cache_trigger(ipv6_udp_pkt()) + test_ipv6_cache_trigger(ipv6_tcp_pkt()) end From 26dc11aab473bc4edecdd710c339395afc7c5c26 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 9 Jan 2017 00:07:41 +0100 Subject: [PATCH 583/631] Set random destination port in IPv4 refresh packets --- src/apps/lwaftr/constants.lua | 1 + src/apps/lwaftr/nh_fwd.lua | 28 ++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/apps/lwaftr/constants.lua b/src/apps/lwaftr/constants.lua index f2aca5c456..b1fbdc1abc 100644 --- a/src/apps/lwaftr/constants.lua +++ b/src/apps/lwaftr/constants.lua @@ -8,6 +8,7 @@ local C = ffi.C proto_icmp = 1 proto_ipv4 = 4 proto_tcp = 6 +proto_udp = 17 ipv6_frag = 44 proto_icmpv6 = 58 diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index 9d351a650c..42adf9281e 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -1,11 +1,14 @@ module(..., package.seeall) +local lwdebug = require("apps.lwaftr.lwdebug") local app = require("core.app") local basic_apps = require("apps.basic.basic_apps") local bit = require("bit") local constants = require("apps.lwaftr.constants") local ethernet = require("lib.protocol.ethernet") local ipsum = require("lib.checksum").ipsum +local tcp = require("lib.protocol.tcp") +local udp = require("lib.protocol.udp") local ipv4 = require("lib.protocol.ipv4") local ipv6 = require("lib.protocol.ipv6") local lib = require("core.lib") @@ -41,6 +44,9 @@ nh_fwd6 = { } } +local proto_tcp = constants.proto_tcp +local proto_udp = constants.proto_udp + local ethernet_header_size = constants.ethernet_header_size local n_ethertype_ipv4 = constants.n_ethertype_ipv4 local proto_ipv4 = constants.proto_ipv4 @@ -114,6 +120,10 @@ local function send_ipv6_cache_trigger (r, pkt, mac) transmit(r, ipv6_cache_trigger(pkt, mac)) end +local function random_port () + return math.random(65535) +end + local function ipv4_cache_trigger (pkt, mac) local ether_hdr = ethernet:new_from_mem(pkt.data, ethernet_header_size) local ip_hdr = ipv4:new_from_mem(pkt.data + ethernet_header_size, pkt.length - ethernet_header_size) @@ -122,8 +132,21 @@ local function ipv4_cache_trigger (pkt, mac) ether_hdr:dst(mac) -- Set a bogus source IP address. ip_hdr:src(n_cache_src_ipv4) + -- Set random port. + local tcp_hdr + local tcp_offset = ethernet_header_size + (ip_hdr:ihl() * 4) + local proto = ip_hdr:protocol() + if proto == proto_tcp then + tcp_hdr = tcp:new_from_mem(pkt.data + tcp_offset, pkt.length - tcp_offset) + payload_offset = tcp_offset + tcp_hdr:sizeof() + elseif proto == proto_udp then + tcp_hdr = udp:new_from_mem(pkt.data + tcp_offset, pkt.length - tcp_offset) + payload_offset = tcp_offset + tcp_hdr:sizeof() + end + tcp_hdr:dst_port(random_port()) -- Recalculate checksum. ip_hdr:checksum() + tcp_hdr:checksum(pkt.data + payload_offset, pkt.length - payload_offset, ip_hdr) return pkt end @@ -192,13 +215,13 @@ function nh_fwd4:push () local pkt = receive(input_vm) local ether_dhost = get_ether_dhost_ptr(pkt) local ipv4_hdr = get_ethernet_payload(pkt) - + if service_mac and ether_equals(ether_dhost, service_mac) then transmit(output_service, pkt) elseif self.cache_refresh_interval > 0 and get_ipv4_src_address(ipv4_hdr) == val_cache_src_ipv4 then -- Our magic cache next-hop resolution packet. Never send this out. - + copy_ether(self.next_hop_mac, ether_dhost) if self.debug then print(("nh_fwd4: learning next-hop '%s'"):format(ethernet:ntop(ether_dhost))) @@ -568,6 +591,7 @@ local function test_ipv4_cache_trigger (pkt) local ip_hdr = ipv4:new_from_mem(refresh_packet.data + ethernet_header_size, pkt.length - ethernet_header_size) assert(ip_hdr:src_eq(n_cache_src_ipv4)) assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) + lwdebug.print_pkt(refresh_packet) end local function test_ipv6_cache_trigger (pkt) From 1299a7454da2bf5db628396dddae1c265687eac6 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sat, 14 Jan 2017 12:11:01 +0100 Subject: [PATCH 584/631] Set random source port in IPv4 payload of IPv6 refresh packets --- src/apps/lwaftr/nh_fwd.lua | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index 42adf9281e..9ec0a31299 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -56,6 +56,7 @@ local o_ipv4_dst_addr = constants.o_ipv4_dst_addr local o_ipv4_src_addr = constants.o_ipv4_src_addr local o_ipv6_next_header = constants.o_ipv6_next_header local o_ipv6_src_addr = constants.o_ipv6_src_addr +local ipv6_fixed_header_size = constants.ipv6_fixed_header_size local n_cache_src_ipv4 = ipv4:pton("169.254.254.254") local val_cache_src_ipv4 = rd32(n_cache_src_ipv4) @@ -93,6 +94,10 @@ local function copy_ether(dst, src) ffi.copy(dst, src, 6) end +local function random_port () + return math.random(65535) +end + -- Set a bogus source IP address fe80::, so we can recognize it later when -- it comes back from the VM. -- @@ -106,12 +111,31 @@ end -- local function ipv6_cache_trigger (pkt, mac) local ether_hdr = ethernet:new_from_mem(pkt.data, ethernet_header_size) - local ip_hdr = ipv6:new_from_mem(pkt.data + ethernet_header_size, pkt.length - ethernet_header_size) + local ipv6_hdr = ipv6:new_from_mem(pkt.data + ethernet_header_size, pkt.length - ethernet_header_size) + local ipv6_payload_offset = ethernet_header_size + ipv6_fixed_header_size + local ipv4_hdr = ipv4:new_from_mem(pkt.data + ipv6_payload_offset, pkt.length - ipv6_payload_offset) -- VM will discard packets not matching its MAC address on the interface. ether_hdr:dst(mac) -- Set a bogus source IP address. - ip_hdr:src(n_cache_src_ipv6) + ipv6_hdr:src(n_cache_src_ipv6) + + -- Set random port. + local tcp_hdr + local tcp_offset = ipv6_payload_offset + ipv4_hdr:ihl() * 4 + local proto = ipv4_hdr:protocol() + local payload_offset + if proto == proto_tcp then + tcp_hdr = tcp:new_from_mem(pkt.data + tcp_offset, pkt.length - tcp_offset) + payload_offset = tcp_offset + tcp_hdr:sizeof() + elseif proto == proto_udp then + tcp_hdr = udp:new_from_mem(pkt.data + tcp_offset, pkt.length - tcp_offset) + payload_offset = tcp_offset + tcp_hdr:sizeof() + end + tcp_hdr:src_port(random_port()) + -- Recalculate checksum. + ipv4_hdr:checksum() + tcp_hdr:checksum(pkt.data + payload_offset, pkt.length - payload_offset, ipv4_hdr) return pkt end @@ -120,10 +144,6 @@ local function send_ipv6_cache_trigger (r, pkt, mac) transmit(r, ipv6_cache_trigger(pkt, mac)) end -local function random_port () - return math.random(65535) -end - local function ipv4_cache_trigger (pkt, mac) local ether_hdr = ethernet:new_from_mem(pkt.data, ethernet_header_size) local ip_hdr = ipv4:new_from_mem(pkt.data + ethernet_header_size, pkt.length - ethernet_header_size) @@ -136,6 +156,7 @@ local function ipv4_cache_trigger (pkt, mac) local tcp_hdr local tcp_offset = ethernet_header_size + (ip_hdr:ihl() * 4) local proto = ip_hdr:protocol() + local payload_offset if proto == proto_tcp then tcp_hdr = tcp:new_from_mem(pkt.data + tcp_offset, pkt.length - tcp_offset) payload_offset = tcp_offset + tcp_hdr:sizeof() From 3249679c893d35b6f81373c9612cb8ef7cde3176 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sat, 14 Jan 2017 12:17:52 +0100 Subject: [PATCH 585/631] Remove unused variables --- src/apps/lwaftr/nh_fwd.lua | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index 9ec0a31299..afe53b121c 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -1,12 +1,9 @@ module(..., package.seeall) -local lwdebug = require("apps.lwaftr.lwdebug") local app = require("core.app") local basic_apps = require("apps.basic.basic_apps") -local bit = require("bit") local constants = require("apps.lwaftr.constants") local ethernet = require("lib.protocol.ethernet") -local ipsum = require("lib.checksum").ipsum local tcp = require("lib.protocol.tcp") local udp = require("lib.protocol.udp") local ipv4 = require("lib.protocol.ipv4") @@ -19,10 +16,8 @@ local ffi = require("ffi") local C = ffi.C local transmit, receive = link.transmit, link.receive -local htons = lib.htons -local rd16, rd32, wr16 = lwutil.rd16, lwutil.rd32, lwutil.wr16 +local rd16, rd32 = lwutil.rd16, lwutil.rd32 local ipv6_equals = lwutil.ipv6_equals -local lshift, band = bit.lshift, bit.band nh_fwd4 = { config = { @@ -51,7 +46,6 @@ local ethernet_header_size = constants.ethernet_header_size local n_ethertype_ipv4 = constants.n_ethertype_ipv4 local proto_ipv4 = constants.proto_ipv4 local ipv6_frag = constants.ipv6_frag -local o_ipv4_checksum = constants.o_ipv4_checksum local o_ipv4_dst_addr = constants.o_ipv4_dst_addr local o_ipv4_src_addr = constants.o_ipv4_src_addr local o_ipv6_next_header = constants.o_ipv6_next_header @@ -612,7 +606,6 @@ local function test_ipv4_cache_trigger (pkt) local ip_hdr = ipv4:new_from_mem(refresh_packet.data + ethernet_header_size, pkt.length - ethernet_header_size) assert(ip_hdr:src_eq(n_cache_src_ipv4)) assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) - lwdebug.print_pkt(refresh_packet) end local function test_ipv6_cache_trigger (pkt) From 78f88f6cdc7497c1dbb121bc1c3774db2071400a Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Sat, 14 Jan 2017 16:38:37 +0100 Subject: [PATCH 586/631] Verify checksum of modified refresh packets --- src/apps/lwaftr/nh_fwd.lua | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/apps/lwaftr/nh_fwd.lua b/src/apps/lwaftr/nh_fwd.lua index afe53b121c..35986ea2d1 100644 --- a/src/apps/lwaftr/nh_fwd.lua +++ b/src/apps/lwaftr/nh_fwd.lua @@ -600,21 +600,31 @@ local function test_ipv6_flow () end local function test_ipv4_cache_trigger (pkt) + local checksum = require("lib.checksum") local ether_dhost = "52:54:00:00:00:01" local refresh_packet = ipv4_cache_trigger(pkt, ethernet:pton(ether_dhost)) local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) - local ip_hdr = ipv4:new_from_mem(refresh_packet.data + ethernet_header_size, pkt.length - ethernet_header_size) + local ip_hdr = ipv4:new_from_mem(refresh_packet.data + ethernet_header_size, + refresh_packet.length - ethernet_header_size) assert(ip_hdr:src_eq(n_cache_src_ipv4)) assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) + assert(checksum.verify_packet(refresh_packet.data + ethernet_header_size, + refresh_packet.length - ethernet_header_size)) end local function test_ipv6_cache_trigger (pkt) + local checksum = require("lib.checksum") local ether_dhost = "52:54:00:00:00:01" local refresh_packet = ipv6_cache_trigger(pkt, ethernet:pton(ether_dhost)) local eth_hdr = ethernet:new_from_mem(refresh_packet.data, ethernet_header_size) - local ip_hdr = ipv6:new_from_mem(refresh_packet.data + ethernet_header_size, pkt.length - ethernet_header_size) + local ip_hdr = ipv6:new_from_mem(refresh_packet.data + ethernet_header_size, + refresh_packet.length - ethernet_header_size) assert(ip_hdr:src_eq(n_cache_src_ipv6)) assert(ethernet:ntop(eth_hdr:dst()) == ether_dhost) + local payload_offset = ethernet_header_size + ipv6_fixed_header_size + local ipv4_pkt = refresh_packet.data + payload_offset + local ipv4_pkt_length = refresh_packet.length - payload_offset + assert(checksum.verify_packet(ipv4_pkt, ipv4_pkt_length)) end local function ipv4_udp_pkt () From 1b192e0cc6eb010e232a9c8ef73208a5031404fa Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Thu, 9 Mar 2017 13:53:31 +0100 Subject: [PATCH 587/631] Improve Python selftests (#763) * Allow running tests in just one subdirectory * Move exit code checks to one place * Add a TestCase superclass for tests needing a long-running process * Remove the sh library, add a TestCase superclass for tests needing a running daemon * Improve test diagnostics --- src/program/lwaftr/tests/lib/test_env.py | 66 +++++++++++++++++-- src/program/lwaftr/tests/selftest.sh | 19 ++++-- .../lwaftr/tests/subcommands/bench_test.py | 24 ++++--- .../lwaftr/tests/subcommands/check_test.py | 23 ++++--- .../lwaftr/tests/subcommands/loadtest_test.py | 36 ++++------ .../lwaftr/tests/subcommands/monitor_test.py | 40 ++++------- .../lwaftr/tests/subcommands/run_test.py | 21 +++--- 7 files changed, 136 insertions(+), 93 deletions(-) diff --git a/src/program/lwaftr/tests/lib/test_env.py b/src/program/lwaftr/tests/lib/test_env.py index 8cbda5708e..809d7f36f3 100644 --- a/src/program/lwaftr/tests/lib/test_env.py +++ b/src/program/lwaftr/tests/lib/test_env.py @@ -4,8 +4,10 @@ import os from pathlib import Path - -from lib import sh +from signal import SIGTERM +from subprocess import PIPE, Popen, TimeoutExpired, check_output +import time +import unittest # Commands run under "sudo" run as root. The root's user PATH should not @@ -21,6 +23,7 @@ BENCHMARK_FILENAME = 'benchtest.csv' # Snabb creates the benchmark file in the current directory BENCHMARK_PATH = Path.cwd() / BENCHMARK_FILENAME +COMMAND_TIMEOUT = 10 def nic_names(): @@ -32,8 +35,63 @@ def tap_name(): Return the first TAP interface name if one found: (tap_iface, None). Return (None, 'No TAP interface available') if none found. """ - output = sh.ip('tuntap', 'list') - tap_iface = output.split(':')[0] + output = check_output(['ip', 'tuntap', 'list']) + tap_iface = output.split(b':')[0] if not tap_iface: return None, 'No TAP interface available' return tap_iface, None + + +class BaseTestCase(unittest.TestCase): + """ + Base class for TestCases. It has a "run_cmd" method and daemon handling, + running a subcommand like "snabb lwaftr run" or "bench". + + Set the daemon args in "cls.daemon_args" to enable the daemon. + Set "self.wait_for_daemon_startup" to True if it needs time to start up. + """ + + # Override these. + daemon_args = () + wait_for_daemon_startup = False + + # Use setUpClass to only setup the daemon once for all tests. + @classmethod + def setUpClass(cls): + if not cls.daemon_args: + return + cls.daemon = Popen(cls.daemon_args, stdout=PIPE, stderr=PIPE) + if cls.wait_for_daemon_startup: + time.sleep(1) + + def run_cmd(self, args): + proc = Popen(args, stdout=PIPE, stderr=PIPE) + try: + output, errput = proc.communicate(timeout=COMMAND_TIMEOUT) + except TimeoutExpired: + proc.kill() + output, errput = proc.communicate() + msg = '\n'.join(('Timeout running command:', str(args), + 'STDOUT', str(output), 'STDERR', str(errput))) + self.fail(msg) + else: + if proc.returncode != 0: + msg = '\n'.join(('Error running command:', str(args), + 'Exit code:', str(proc.returncode), + 'STDOUT', str(output), 'STDERR', str(errput))) + self.fail(msg) + return output + + @classmethod + def tearDownClass(cls): + if not cls.daemon_args: + return + ret_code = cls.daemon.poll() + if ret_code is None: + cls.daemon.terminate() + ret_code = cls.daemon.wait() + if ret_code not in (0, -SIGTERM): + print('Error running daemon:', cls.daemon.args) + print('Exit code:', ret_code) + print('STDOUT\n', cls.daemon.stdout.read()) + print('STDERR\n', cls.daemon.stderr.read()) diff --git a/src/program/lwaftr/tests/selftest.sh b/src/program/lwaftr/tests/selftest.sh index 23b4cd769b..88af5b6ebb 100755 --- a/src/program/lwaftr/tests/selftest.sh +++ b/src/program/lwaftr/tests/selftest.sh @@ -1,14 +1,23 @@ #!/usr/bin/env bash +# Entry point for Python tests. +# # Make it work from wherever this script is called, and let tests know. export TESTS_DIR=`dirname "$0"` +export PYTHONPATH=${TESTS_DIR} -# Entry point for Python tests. -# -# Start discovery from this script's directory, the root of the "tests" subtree. -# Find unittests in all Python files ending with "_test.py". +# Only run tests in the passed subdirectory of $TESTS_DIR. +if [[ -n $1 ]]; then + START_DIR=${TESTS_DIR}/$1/ +else + START_DIR=${TESTS_DIR} +fi + +# Start discovery from this script's directory, the root of the "tests" subtree, +# or one of its subdirectories, if passed as first argument to this script. +# Look for unittests in all files whose name ends with "_test.py". # List all executed tests, don't show just dots. python3 -m unittest discover \ - --start-directory "${TESTS_DIR}" \ + --start-directory "${START_DIR}" \ --pattern "*_test.py" \ --verbose diff --git a/src/program/lwaftr/tests/subcommands/bench_test.py b/src/program/lwaftr/tests/subcommands/bench_test.py index 90ff6aa6fc..852837fa2e 100644 --- a/src/program/lwaftr/tests/subcommands/bench_test.py +++ b/src/program/lwaftr/tests/subcommands/bench_test.py @@ -4,25 +4,23 @@ import unittest -from lib import sh -from lib.test_env import ( - BENCHMARK_FILENAME, BENCHMARK_PATH, DATA_DIR, BENCHDATA_DIR, SNABB_CMD) +from lib.test_env import (BENCHMARK_FILENAME, BENCHMARK_PATH, DATA_DIR, + BENCHDATA_DIR, SNABB_CMD, BaseTestCase) -class TestBench(unittest.TestCase): +class TestBench(BaseTestCase): cmd_args = ( - SNABB_CMD, 'lwaftr', 'bench', + str(SNABB_CMD), 'lwaftr', 'bench', '--duration', '0.1', '--bench-file', BENCHMARK_FILENAME, - DATA_DIR / 'icmp_on_fail.conf', - BENCHDATA_DIR / 'ipv4-0550.pcap', - BENCHDATA_DIR / 'ipv6-0550.pcap', + str(DATA_DIR / 'icmp_on_fail.conf'), + str(BENCHDATA_DIR / 'ipv4-0550.pcap'), + str(BENCHDATA_DIR / 'ipv6-0550.pcap'), ) def execute_bench_test(self, cmd_args): - output = sh.sudo(*cmd_args) - self.assertEqual(output.exit_code, 0) + self.run_cmd(cmd_args) self.assertTrue(BENCHMARK_PATH.is_file(), 'Cannot find {}'.format(BENCHMARK_PATH)) BENCHMARK_PATH.unlink() @@ -31,9 +29,9 @@ def test_bench_standard(self): self.execute_bench_test(self.cmd_args) def test_bench_reconfigurable(self): - reconf_cmd_args = list(self.cmd_args) - reconf_cmd_args.insert(3, '--reconfigurable') - self.execute_bench_test(reconf_cmd_args) + reconf_args = list(self.cmd_args) + reconf_args.insert(3, '--reconfigurable') + self.execute_bench_test(reconf_args) if __name__ == '__main__': diff --git a/src/program/lwaftr/tests/subcommands/check_test.py b/src/program/lwaftr/tests/subcommands/check_test.py index 07f5c5dbf4..d4670b64f5 100644 --- a/src/program/lwaftr/tests/subcommands/check_test.py +++ b/src/program/lwaftr/tests/subcommands/check_test.py @@ -4,31 +4,30 @@ import unittest -from lib import sh -from lib.test_env import COUNTERS_DIR, DATA_DIR, SNABB_CMD +from lib.test_env import COUNTERS_DIR, DATA_DIR, SNABB_CMD, BaseTestCase -class TestCheck(unittest.TestCase): +class TestCheck(BaseTestCase): cmd_args = ( - SNABB_CMD, 'lwaftr', 'check', - DATA_DIR / 'icmp_on_fail.conf', - DATA_DIR / 'empty.pcap', DATA_DIR / 'empty.pcap', + str(SNABB_CMD), 'lwaftr', 'check', + str(DATA_DIR / 'icmp_on_fail.conf'), + str(DATA_DIR / 'empty.pcap'), str(DATA_DIR / 'empty.pcap'), '/dev/null', '/dev/null', - COUNTERS_DIR / 'empty.lua', + str(COUNTERS_DIR / 'empty.lua'), ) def execute_check_test(self, cmd_args): - output = sh.sudo(*cmd_args) - self.assertEqual(output.exit_code, 0) + self.run_cmd(cmd_args) + # run_cmd checks the exit code and fails the test if it is not zero. def test_check_standard(self): self.execute_check_test(self.cmd_args) def test_check_on_a_stick(self): - onastick_cmd_args = list(self.cmd_args) - onastick_cmd_args.insert(3, '--on-a-stick') - self.execute_check_test(onastick_cmd_args) + onastick_args = list(self.cmd_args) + onastick_args.insert(3, '--on-a-stick') + self.execute_check_test(onastick_args) if __name__ == '__main__': diff --git a/src/program/lwaftr/tests/subcommands/loadtest_test.py b/src/program/lwaftr/tests/subcommands/loadtest_test.py index 08ae622f2e..4538b16bdb 100644 --- a/src/program/lwaftr/tests/subcommands/loadtest_test.py +++ b/src/program/lwaftr/tests/subcommands/loadtest_test.py @@ -8,25 +8,24 @@ import unittest -from lib import sh -from lib.test_env import BENCHDATA_DIR, DATA_DIR, SNABB_CMD, nic_names +from lib.test_env import ( + BENCHDATA_DIR, DATA_DIR, SNABB_CMD, BaseTestCase, nic_names) SNABB_PCI0, SNABB_PCI1 = nic_names() @unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') -class TestLoadtest(unittest.TestCase): +class TestLoadtest(BaseTestCase): - run_cmd_args = ( - SNABB_CMD, 'lwaftr', 'run', + daemon_args = ( + str(SNABB_CMD), 'lwaftr', 'run', '--bench-file', '/dev/null', - '--conf', DATA_DIR / 'icmp_on_fail.conf', + '--conf', str(DATA_DIR / 'icmp_on_fail.conf'), '--on-a-stick', SNABB_PCI0, ) - - loadtest_cmd_args = ( - SNABB_CMD, 'lwaftr', 'loadtest', + loadtest_args = ( + str(SNABB_CMD), 'lwaftr', 'loadtest', '--bench-file', '/dev/null', # Something quick and easy. '--program', 'ramp_up', @@ -34,22 +33,15 @@ class TestLoadtest(unittest.TestCase): '--duration', '0.1', '--bitrate', '0.2e8', # Just one card for on-a-stick mode. - BENCHDATA_DIR / 'ipv4_and_ipv6_stick_imix.pcap', 'ALL', 'ALL', SNABB_PCI1, + str(BENCHDATA_DIR / 'ipv4_and_ipv6_stick_imix.pcap'), 'ALL', 'ALL', + SNABB_PCI1, ) - - # Use setUpClass to only setup the "run" daemon once for all tests. - @classmethod - def setUpClass(cls): - cls.run_cmd = sh.sudo(*cls.run_cmd_args, _bg=True) + wait_for_daemon_startup = True def test_loadtest(self): - output = sh.sudo(*self.loadtest_cmd_args) - self.assertEqual(output.exit_code, 0) - self.assertTrue(len(output.splitlines()) > 10) - - @classmethod - def tearDownClass(cls): - cls.run_cmd.terminate() + output = self.run_cmd(self.loadtest_args) + self.assertGreater(len(output.splitlines()), 10, + b'\n'.join((b'OUTPUT', output))) if __name__ == '__main__': diff --git a/src/program/lwaftr/tests/subcommands/monitor_test.py b/src/program/lwaftr/tests/subcommands/monitor_test.py index 35b8972ed8..fe8c605aa2 100644 --- a/src/program/lwaftr/tests/subcommands/monitor_test.py +++ b/src/program/lwaftr/tests/subcommands/monitor_test.py @@ -7,8 +7,7 @@ import unittest -from lib import sh -from lib.test_env import DATA_DIR, SNABB_CMD, nic_names, tap_name +from lib.test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names, tap_name SNABB_PCI0 = nic_names()[0] @@ -17,37 +16,26 @@ @unittest.skipUnless(SNABB_PCI0, 'NIC not configured') @unittest.skipUnless(TAP_IFACE, tap_err_msg) -class TestMonitor(unittest.TestCase): +class TestMonitor(BaseTestCase): - run_cmd_args = ( - SNABB_CMD, 'lwaftr', 'run', - '--name', 'monitor_test', + daemon_args = ( + str(SNABB_CMD), 'lwaftr', 'run', '--bench-file', '/dev/null', - '--conf', DATA_DIR / 'icmp_on_fail.conf', + '--conf', str(DATA_DIR / 'icmp_on_fail.conf'), '--on-a-stick', SNABB_PCI0, '--mirror', TAP_IFACE, ) - - monitor_cmd_args = ( - SNABB_CMD, 'lwaftr', 'monitor', - '--name', 'monitor_test', - 'all', - ) - - # Use setUpClass to only setup the "run" daemon once for all tests. - @classmethod - def setUpClass(cls): - cls.run_cmd = sh.sudo(*cls.run_cmd_args, _bg=True) + monitor_args = (str(SNABB_CMD), 'lwaftr', 'monitor', 'all') + wait_for_daemon_startup = True def test_monitor(self): - output = sh.sudo(*self.monitor_cmd_args) - self.assertEqual(output.exit_code, 0) - self.assertIn('Mirror address set', output) - self.assertIn('255.255.255.255', output) - - @classmethod - def tearDownClass(cls): - cls.run_cmd.terminate() + monitor_args = list(self.monitor_args) + monitor_args.append(str(self.daemon.pid)) + output = self.run_cmd(monitor_args) + self.assertIn(b'Mirror address set', output, + b'\n'.join((b'OUTPUT', output))) + self.assertIn(b'255.255.255.255', output, + b'\n'.join((b'OUTPUT', output))) if __name__ == '__main__': diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py index a697dab8bc..751ee438e0 100644 --- a/src/program/lwaftr/tests/subcommands/run_test.py +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -4,37 +4,36 @@ import unittest -from lib import sh -from lib.test_env import DATA_DIR, SNABB_CMD, nic_names +from lib.test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names SNABB_PCI0, SNABB_PCI1 = nic_names() @unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') -class TestRun(unittest.TestCase): +class TestRun(BaseTestCase): cmd_args = ( - SNABB_CMD, 'lwaftr', 'run', + str(SNABB_CMD), 'lwaftr', 'run', '--duration', '0.1', '--bench-file', '/dev/null', - '--conf', DATA_DIR / 'icmp_on_fail.conf', + '--conf', str(DATA_DIR / 'icmp_on_fail.conf'), '--v4', SNABB_PCI0, '--v6', SNABB_PCI1, ) def execute_run_test(self, cmd_args): - output = sh.sudo(*cmd_args) - self.assertEqual(output.exit_code, 0) - self.assertTrue(len(output.splitlines()) > 1) + output = self.run_cmd(cmd_args) + self.assertIn(b'link report', output, + b'\n'.join((b'OUTPUT', output))) def test_run_standard(self): self.execute_run_test(self.cmd_args) def test_run_reconfigurable(self): - reconf_cmd_args = list(self.cmd_args) - reconf_cmd_args.insert(3, '--reconfigurable') - self.execute_run_test(reconf_cmd_args) + reconf_args = list(self.cmd_args) + reconf_args.insert(3, '--reconfigurable') + self.execute_run_test(reconf_args) if __name__ == '__main__': From 22f4a7fcce5ee2fa2b2b61a8d23c6e9138170654 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 13 Mar 2017 15:33:00 +0100 Subject: [PATCH 588/631] Remove the 'sh' Python library (#771) --- src/program/lwaftr/tests/lib/__init__.py | 0 src/program/lwaftr/tests/lib/sh.py | 3399 ----------------- .../lwaftr/tests/subcommands/bench_test.py | 2 +- .../lwaftr/tests/subcommands/check_test.py | 2 +- .../lwaftr/tests/subcommands/loadtest_test.py | 3 +- .../lwaftr/tests/subcommands/monitor_test.py | 2 +- .../lwaftr/tests/subcommands/run_test.py | 2 +- .../lwaftr/tests/{lib => }/test_env.py | 0 8 files changed, 5 insertions(+), 3405 deletions(-) delete mode 100644 src/program/lwaftr/tests/lib/__init__.py delete mode 100644 src/program/lwaftr/tests/lib/sh.py rename src/program/lwaftr/tests/{lib => }/test_env.py (100%) diff --git a/src/program/lwaftr/tests/lib/__init__.py b/src/program/lwaftr/tests/lib/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/program/lwaftr/tests/lib/sh.py b/src/program/lwaftr/tests/lib/sh.py deleted file mode 100644 index 780e7613e6..0000000000 --- a/src/program/lwaftr/tests/lib/sh.py +++ /dev/null @@ -1,3399 +0,0 @@ -""" -http://amoffat.github.io/sh/ -""" -#=============================================================================== -# Copyright (C) 2011-2017 by Andrew Moffat -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -#=============================================================================== - - -__version__ = "1.12.9" -__project_url__ = "https://github.com/amoffat/sh" - - -import platform - -if "windows" in platform.system().lower(): # pragma: no cover - raise ImportError("sh %s is currently only supported on linux and osx. \ -please install pbs 0.110 (http://pypi.python.org/pypi/pbs) for windows \ -support." % __version__) - - -import sys -IS_PY3 = sys.version_info[0] == 3 -MINOR_VER = sys.version_info[1] -IS_PY26 = sys.version_info[0] == 2 and MINOR_VER == 6 - -import traceback -import os -import re -import time -import getpass -from types import ModuleType, GeneratorType -from functools import partial -import inspect -import tempfile -import stat -import glob as glob_module -import ast -from contextlib import contextmanager -import pwd -import errno -from io import UnsupportedOperation - -from locale import getpreferredencoding -DEFAULT_ENCODING = getpreferredencoding() or "UTF-8" - -# normally i would hate this idea of using a global to signify whether we are -# running tests, because it breaks the assumption that what is running in the -# tests is what will run live, but we ONLY use this in a place that has no -# serious side-effects that could change anything. as long as we do that, it -# should be ok -RUNNING_TESTS = bool(int(os.environ.get("SH_TESTS_RUNNING", "0"))) - -if IS_PY3: - from io import StringIO - ioStringIO = StringIO - from io import BytesIO as cStringIO - iocStringIO = cStringIO - from queue import Queue, Empty - - # for some reason, python 3.1 removed the builtin "callable", wtf - if not hasattr(__builtins__, "callable"): - def callable(ob): - return hasattr(ob, "__call__") -else: - from StringIO import StringIO - from cStringIO import OutputType as cStringIO - from io import StringIO as ioStringIO - from io import BytesIO as iocStringIO - from Queue import Queue, Empty - -IS_OSX = platform.system() == "Darwin" -THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -SH_LOGGER_NAME = __name__ - - -import errno -import pty -import termios -import signal -import gc -import select -import threading -import tty -import fcntl -import struct -import resource -from collections import deque -import logging -import weakref - - -# a re-entrant lock for pushd. this way, multiple threads that happen to use -# pushd will all see the current working directory for the duration of the -# with-context -PUSHD_LOCK = threading.RLock() - - -if hasattr(inspect, "signature"): - def get_num_args(fn): - return len(inspect.signature(fn).parameters) -else: - def get_num_args(fn): - return len(inspect.getargspec(fn).args) - -if IS_PY3: - raw_input = input - unicode = str - basestring = str - long = int - - -_unicode_methods = set(dir(unicode())) - - -def encode_to_py3bytes_or_py2str(s): - """ takes anything and attempts to return a py2 string or py3 bytes. this - is typically used when creating command + arguments to be executed via - os.exec* """ - - fallback_encoding = "utf8" - - if IS_PY3: - # if we're already bytes, do nothing - if isinstance(s, bytes): - pass - else: - s = str(s) - try: - s = bytes(s, DEFAULT_ENCODING) - except UnicodeEncodeError: - s = bytes(s, fallback_encoding) - else: - # attempt to convert the thing to unicode from the system's encoding - try: - s = unicode(s, DEFAULT_ENCODING) - # if the thing is already unicode, or it's a number, it can't be - # coerced to unicode with an encoding argument, but if we leave out - # the encoding argument, it will convert it to a string, then to unicode - except TypeError: - s = unicode(s) - - # now that we have guaranteed unicode, encode to our system encoding, - # but attempt to fall back to something - try: - s = s.encode(DEFAULT_ENCODING) - except: - s = s.encode(fallback_encoding, "replace") - return s - - -def _indent_text(text, num=4): - lines = [] - for line in text.split("\n"): - line = (" " * num) + line - lines.append(line) - return "\n".join(lines) - - -class ForkException(Exception): - def __init__(self, orig_exc): - tmpl = """ - -Original exception: -=================== - -%s -""" - msg = tmpl % _indent_text(orig_exc) - Exception.__init__(self, msg) - - -class ErrorReturnCodeMeta(type): - """ a metaclass which provides the ability for an ErrorReturnCode (or - derived) instance, imported from one sh module, to be considered the - subclass of ErrorReturnCode from another module. this is mostly necessary - in the tests, where we do assertRaises, but the ErrorReturnCode that the - program we're testing throws may not be the same class that we pass to - assertRaises - """ - def __subclasscheck__(self, o): - other_bases = set([b.__name__ for b in o.__bases__]) - return self.__name__ in other_bases or o.__name__ == self.__name__ - - -class ErrorReturnCode(Exception): - __metaclass__ = ErrorReturnCodeMeta - - """ base class for all exceptions as a result of a command's exit status - being deemed an error. this base class is dynamically subclassed into - derived classes with the format: ErrorReturnCode_NNN where NNN is the exit - code number. the reason for this is it reduces boiler plate code when - testing error return codes: - - try: - some_cmd() - except ErrorReturnCode_12: - print("couldn't do X") - - vs: - try: - some_cmd() - except ErrorReturnCode as e: - if e.exit_code == 12: - print("couldn't do X") - - it's not much of a savings, but i believe it makes the code easier to read """ - - truncate_cap = 750 - - def __init__(self, full_cmd, stdout, stderr, truncate=True): - self.full_cmd = full_cmd - self.stdout = stdout - self.stderr = stderr - - exc_stdout = self.stdout - if truncate: - exc_stdout = exc_stdout[:self.truncate_cap] - out_delta = len(self.stdout) - len(exc_stdout) - if out_delta: - exc_stdout += ("... (%d more, please see e.stdout)" % out_delta).encode() - - exc_stderr = self.stderr - if truncate: - exc_stderr = exc_stderr[:self.truncate_cap] - err_delta = len(self.stderr) - len(exc_stderr) - if err_delta: - exc_stderr += ("... (%d more, please see e.stderr)" % err_delta).encode() - - msg_tmpl = unicode("\n\n RAN: {cmd}\n\n STDOUT:\n{stdout}\n\n STDERR:\n{stderr}") - - msg = msg_tmpl.format( - cmd=self.full_cmd, - stdout=exc_stdout.decode(DEFAULT_ENCODING, "replace"), - stderr=exc_stderr.decode(DEFAULT_ENCODING, "replace") - ) - - super(ErrorReturnCode, self).__init__(msg) - - -class SignalException(ErrorReturnCode): pass -class TimeoutException(Exception): - """ the exception thrown when a command is killed because a specified - timeout (via _timeout) was hit """ - def __init__(self, exit_code): - self.exit_code = exit_code - super(Exception, self).__init__() - -SIGNALS_THAT_SHOULD_THROW_EXCEPTION = ( - signal.SIGABRT, - signal.SIGBUS, - signal.SIGFPE, - signal.SIGILL, - signal.SIGINT, - signal.SIGKILL, - signal.SIGPIPE, - signal.SIGQUIT, - signal.SIGSEGV, - signal.SIGTERM, - signal.SIGSYS, -) - - -# we subclass AttributeError because: -# https://github.com/ipython/ipython/issues/2577 -# https://github.com/amoffat/sh/issues/97#issuecomment-10610629 -class CommandNotFound(AttributeError): pass - - - - -rc_exc_regex = re.compile("(ErrorReturnCode|SignalException)_((\d+)|SIG[a-zA-Z]+)") -rc_exc_cache = {} - -SIGNAL_MAPPING = {} -for k,v in signal.__dict__.items(): - if re.match(r"SIG[a-zA-Z]+", k): - SIGNAL_MAPPING[v] = k - - -def get_exc_from_name(name): - """ takes an exception name, like: - - ErrorReturnCode_1 - SignalException_9 - SignalException_SIGHUP - - and returns the corresponding exception. this is primarily used for - importing exceptions from sh into user code, for instance, to capture those - exceptions """ - - exc = None - try: - return rc_exc_cache[name] - except KeyError: - m = rc_exc_regex.match(name) - if m: - base = m.group(1) - rc_or_sig_name = m.group(2) - - if base == "SignalException": - try: - rc = -int(rc_or_sig_name) - except ValueError: - rc = -getattr(signal, rc_or_sig_name) - else: - rc = int(rc_or_sig_name) - - exc = get_rc_exc(rc) - return exc - - -def get_rc_exc(rc): - """ takes a exit code or negative signal number and produces an exception - that corresponds to that return code. positive return codes yield - ErrorReturnCode exception, negative return codes yield SignalException - - we also cache the generated exception so that only one signal of that type - exists, preserving identity """ - - try: - return rc_exc_cache[rc] - except KeyError: - pass - - if rc > 0: - name = "ErrorReturnCode_%d" % rc - base = ErrorReturnCode - else: - signame = SIGNAL_MAPPING[abs(rc)] - name = "SignalException_" + signame - base = SignalException - - exc = ErrorReturnCodeMeta(name, (base,), {"exit_code": rc}) - rc_exc_cache[rc] = exc - return exc - - - -# we monkey patch glob. i'm normally generally against monkey patching, but i -# decided to do this really un-intrusive patch because we need a way to detect -# if a list that we pass into an sh command was generated from glob. the reason -# being that glob returns an empty list if a pattern is not found, and so -# commands will treat the empty list as no arguments, which can be a problem, -# ie: -# -# ls(glob("*.ojfawe")) -# -# ^ will show the contents of your home directory, because it's essentially -# running ls([]) which, as a process, is just "ls". -# -# so we subclass list and monkey patch the glob function. nobody should be the -# wiser, but we'll have results that we can make some determinations on -_old_glob = glob_module.glob - -class GlobResults(list): - def __init__(self, path, results): - self.path = path - list.__init__(self, results) - -def glob(path, *args, **kwargs): - expanded = GlobResults(path, _old_glob(path, *args, **kwargs)) - return expanded - -glob_module.glob = glob - - - - -def which(program, paths=None): - """ takes a program name or full path, plus an optional collection of search - paths, and returns the full path of the requested executable. if paths is - specified, it is the entire list of search paths, and the PATH env is not - used at all. otherwise, PATH env is used to look for the program """ - - def is_exe(fpath): - return (os.path.exists(fpath) and - os.access(fpath, os.X_OK) and - os.path.isfile(os.path.realpath(fpath))) - - found_path = None - fpath, fname = os.path.split(program) - - # if there's a path component, then we've specified a path to the program, - # and we should just test if that program is executable. if it is, return - if fpath: - if is_exe(program): - found_path = program - - # otherwise, we've just passed in the program name, and we need to search - # the paths to find where it actually lives - else: - paths_to_search = [] - - if isinstance(paths, (tuple, list)): - paths_to_search.extend(paths) - else: - env_paths = os.environ.get("PATH", "").split(os.pathsep) - paths_to_search.extend(env_paths) - - for path in paths_to_search: - exe_file = os.path.join(path, program) - if is_exe(exe_file): - found_path = exe_file - break - - return found_path - - -def resolve_command_path(program): - path = which(program) - if not path: - # our actual command might have a dash in it, but we can't call - # that from python (we have to use underscores), so we'll check - # if a dash version of our underscore command exists and use that - # if it does - if "_" in program: - path = which(program.replace("_", "-")) - if not path: - return None - return path - - -def resolve_command(name, baked_args=None): - path = resolve_command_path(name) - cmd = None - if path: - cmd = Command(path) - if baked_args: - cmd = cmd.bake(**baked_args) - return cmd - - - - -class Logger(object): - """ provides a memory-inexpensive logger. a gotcha about python's builtin - logger is that logger objects are never garbage collected. if you create a - thousand loggers with unique names, they'll sit there in memory until your - script is done. with sh, it's easy to create loggers with unique names if - we want our loggers to include our command arguments. for example, these - are all unique loggers: - - ls -l - ls -l /tmp - ls /tmp - - so instead of creating unique loggers, and without sacrificing logging - output, we use this class, which maintains as part of its state, the logging - "context", which will be the very unique name. this allows us to get a - logger with a very general name, eg: "command", and have a unique name - appended to it via the context, eg: "ls -l /tmp" """ - def __init__(self, name, context=None): - self.name = name - self.log = logging.getLogger("%s.%s" % (SH_LOGGER_NAME, name)) - self.set_context(context) - - def _format_msg(self, msg, *args): - if self.context: - msg = "%s: %s" % (self.context, msg) - return msg % args - - def set_context(self, context): - if context: - context = context.replace("%", "%%") - self.context = context or "" - - def get_child(self, name, context): - new_name = self.name + "." + name - new_context = self.context + "." + context - l = Logger(new_name, new_context) - return l - - def info(self, msg, *args): - self.log.info(self._format_msg(msg, *args)) - - def debug(self, msg, *args): - self.log.debug(self._format_msg(msg, *args)) - - def error(self, msg, *args): - self.log.error(self._format_msg(msg, *args)) - - def exception(self, msg, *args): - self.log.exception(self._format_msg(msg, *args)) - - -def default_logger_str(cmd, call_args, pid=None): - if pid: - s = "" % (cmd, pid) - else: - s = "" % cmd - return s - - - -class RunningCommand(object): - """ this represents an executing Command object. it is returned as the - result of __call__() being executed on a Command instance. this creates a - reference to a OProc instance, which is a low-level wrapper around the - process that was exec'd - - this is the class that gets manipulated the most by user code, and so it - implements various convenience methods and logical mechanisms for the - underlying process. for example, if a user tries to access a - backgrounded-process's stdout/err, the RunningCommand object is smart enough - to know to wait() on the process to finish first. and when the process - finishes, RunningCommand is smart enough to translate exit codes to - exceptions. """ - - # these are attributes that we allow to passthrough to OProc for - _OProc_attr_whitelist = set(( - "signal", - "terminate", - "kill", - "kill_group", - "signal_group", - "pid", - "sid", - "pgid", - "ctty", - - "input_thread_exc", - "output_thread_exc", - "bg_thread_exc", - )) - - def __init__(self, cmd, call_args, stdin, stdout, stderr): - """ - cmd is an array, where each element is encoded as bytes (PY3) or str - (PY2) - """ - - # self.ran is used for auditing what actually ran. for example, in - # exceptions, or if you just want to know what was ran after the - # command ran - # - # here we're making a consistent unicode string out if our cmd. - # we're also assuming (correctly, i think) that the command and its - # arguments are the encoding we pass into _encoding, which falls back to - # the system's encoding - enc = call_args["encoding"] - self.ran = " ".join([arg.decode(enc, "ignore") for arg in cmd]) - - self.call_args = call_args - self.cmd = cmd - - self.process = None - self._process_completed = False - should_wait = True - spawn_process = True - - # this is used to track if we've already raised StopIteration, and if we - # have, raise it immediately again if the user tries to call next() on - # us. https://github.com/amoffat/sh/issues/273 - self._stopped_iteration = False - - # with contexts shouldn't run at all yet, they prepend - # to every command in the context - if call_args["with"]: - spawn_process = False - get_prepend_stack().append(self) - - - if call_args["piped"] or call_args["iter"] or call_args["iter_noblock"]: - should_wait = False - - # we're running in the background, return self and let us lazily - # evaluate - if call_args["bg"]: - should_wait = False - - # redirection - if call_args["err_to_out"]: - stderr = OProc.STDOUT - - done_callback = call_args["done"] - if done_callback: - call_args["done"] = partial(done_callback, self) - - - # set up which stream should write to the pipe - # TODO, make pipe None by default and limit the size of the Queue - # in oproc.OProc - pipe = OProc.STDOUT - if call_args["iter"] == "out" or call_args["iter"] is True: - pipe = OProc.STDOUT - elif call_args["iter"] == "err": - pipe = OProc.STDERR - - if call_args["iter_noblock"] == "out" or call_args["iter_noblock"] is True: - pipe = OProc.STDOUT - elif call_args["iter_noblock"] == "err": - pipe = OProc.STDERR - - # there's currently only one case where we wouldn't spawn a child - # process, and that's if we're using a with-context with our command - self._spawned_and_waited = False - if spawn_process: - log_str_factory = call_args["log_msg"] or default_logger_str - logger_str = log_str_factory(self.ran, call_args) - self.log = Logger("command", logger_str) - - self.log.info("starting process") - - if should_wait: - self._spawned_and_waited = True - - # this lock is needed because of a race condition where a background - # thread, created in the OProc constructor, may try to access - # self.process, but it has not been assigned yet - process_assign_lock = threading.Lock() - with process_assign_lock: - self.process = OProc(self, self.log, cmd, stdin, stdout, stderr, - self.call_args, pipe, process_assign_lock) - - logger_str = log_str_factory(self.ran, call_args, self.process.pid) - self.log.set_context(logger_str) - self.log.info("process started") - - if should_wait: - self.wait() - - - def wait(self): - """ waits for the running command to finish. this is called on all - running commands, eventually, except for ones that run in the background - """ - if not self._process_completed: - self._process_completed = True - - exit_code = self.process.wait() - if self.process.timed_out: - # if we timed out, our exit code represents a signal, which is - # negative, so let's make it positive to store in our - # TimeoutException - raise TimeoutException(-exit_code) - - else: - self.handle_command_exit_code(exit_code) - - # if an iterable command is using an instance of OProc for its stdin, - # wait on it. the process is probably set to "piped", which means it - # won't be waited on, which means exceptions won't propagate up to the - # main thread. this allows them to bubble up - if self.process._stdin_process: - self.process._stdin_process.command.wait() - - self.log.info("process completed") - return self - - - def handle_command_exit_code(self, code): - """ here we determine if we had an exception, or an error code that we - weren't expecting to see. if we did, we create and raise an exception - """ - exc_class = get_exc_exit_code_would_raise(code, self.call_args["ok_code"]) - if exc_class: - exc = exc_class(self.ran, self.process.stdout, self.process.stderr, - self.call_args["truncate_exc"]) - raise exc - - - @property - def stdout(self): - self.wait() - return self.process.stdout - - @property - def stderr(self): - self.wait() - return self.process.stderr - - @property - def exit_code(self): - self.wait() - return self.process.exit_code - - - def __len__(self): - return len(str(self)) - - def __enter__(self): - """ we don't actually do anything here because anything that should have - been done would have been done in the Command.__call__ call. - essentially all that has to happen is the comand be pushed on the - prepend stack. """ - pass - - def __iter__(self): - return self - - def next(self): - """ allow us to iterate over the output of our command """ - - if self._stopped_iteration: - raise StopIteration() - - # we do this because if get blocks, we can't catch a KeyboardInterrupt - # so the slight timeout allows for that. - while True: - try: - chunk = self.process._pipe_queue.get(True, 0.001) - except Empty: - if self.call_args["iter_noblock"]: - return errno.EWOULDBLOCK - else: - if chunk is None: - self.wait() - self._stopped_iteration = True - raise StopIteration() - try: - return chunk.decode(self.call_args["encoding"], - self.call_args["decode_errors"]) - except UnicodeDecodeError: - return chunk - - - # python 3 - __next__ = next - - def __exit__(self, typ, value, traceback): - if self.call_args["with"] and get_prepend_stack(): - get_prepend_stack().pop() - - def __str__(self): - """ in python3, should return unicode. in python2, should return a - string of bytes """ - if IS_PY3: - return self.__unicode__() - else: - return unicode(self).encode(self.call_args["encoding"]) - - def __unicode__(self): - """ a magic method defined for python2. calling unicode() on a - RunningCommand object will call this """ - if self.process and self.stdout: - return self.stdout.decode(self.call_args["encoding"], - self.call_args["decode_errors"]) - elif IS_PY3: - return "" - else: - return unicode("") - - def __eq__(self, other): - return unicode(self) == unicode(other) - __hash__ = None # Avoid DeprecationWarning in Python < 3 - - def __contains__(self, item): - return item in str(self) - - def __getattr__(self, p): - # let these three attributes pass through to the OProc object - if p in self._OProc_attr_whitelist: - if self.process: - return getattr(self.process, p) - else: - raise AttributeError - - # see if strings have what we're looking for. we're looking at the - # method names explicitly because we don't want to evaluate self unless - # we absolutely have to, the reason being, in python2, hasattr swallows - # exceptions, and if we try to run hasattr on a command that failed and - # is being run with _iter=True, the command will be evaluated, throw an - # exception, but hasattr will discard it - if p in _unicode_methods: - return getattr(unicode(self), p) - - raise AttributeError - - def __repr__(self): - """ in python3, should return unicode. in python2, should return a - string of bytes """ - try: - return str(self) - except UnicodeDecodeError: - if self.process: - if self.stdout: - return repr(self.stdout) - return repr("") - - def __long__(self): - return long(str(self).strip()) - - def __float__(self): - return float(str(self).strip()) - - def __int__(self): - return int(str(self).strip()) - - - -def output_redirect_is_filename(out): - return isinstance(out, basestring) - - -def get_prepend_stack(): - tl = Command.thread_local - if not hasattr(tl, "_prepend_stack"): - tl._prepend_stack = [] - return tl._prepend_stack - - -def special_kwarg_validator(kwargs, invalid_list): - s1 = set(kwargs.keys()) - invalid_args = [] - - for args in invalid_list: - - if callable(args): - fn = args - ret = fn(kwargs) - invalid_args.extend(ret) - - else: - args, error_msg = args - - if s1.issuperset(args): - invalid_args.append((args, error_msg)) - - return invalid_args - - -def get_fileno(ob): - # in py2, this will return None. in py3, it will return an method that - # raises when called - fileno_meth = getattr(ob, "fileno", None) - - fileno = None - if fileno_meth: - # py3 StringIO objects will report a fileno, but calling it will raise - # an exception - try: - fileno = fileno_meth() - except UnsupportedOperation: - pass - elif isinstance(ob, (int,long)) and ob >= 0: - fileno = ob - - return fileno - - -def ob_is_tty(ob): - """ checks if an object (like a file-like object) is a tty. """ - fileno = get_fileno(ob) - is_tty = False - if fileno: - is_tty = os.isatty(fileno) - return is_tty - -def ob_is_pipe(ob): - fileno = get_fileno(ob) - is_pipe = False - if fileno: - fd_stat = os.fstat(fileno) - is_pipe = stat.S_ISFIFO(fd_stat.st_mode) - return is_pipe - - -def tty_in_validator(kwargs): - pairs = (("tty_in", "in"), ("tty_out", "out")) - invalid = [] - for tty, std in pairs: - if tty in kwargs and ob_is_tty(kwargs.get(std, None)): - args = (tty, std) - error = "`_%s` is a TTY already, so so it doesn't make sense \ -to set up a TTY with `_%s`" % (std, tty) - invalid.append((args, error)) - - return invalid - -def bufsize_validator(kwargs): - """ a validator to prevent a user from saying that they want custom - buffering when they're using an in/out object that will be os.dup'd to the - process, and has its own buffering. an example is a pipe or a tty. it - doesn't make sense to tell them to have a custom buffering, since the os - controls this. """ - invalid = [] - - in_ob = kwargs.get("in", None) - out_ob = kwargs.get("out", None) - - in_buf = kwargs.get("in_bufsize", None) - out_buf = kwargs.get("out_bufsize", None) - - in_no_buf = ob_is_tty(in_ob) or ob_is_pipe(in_ob) - out_no_buf = ob_is_tty(out_ob) or ob_is_pipe(out_ob) - - err = "Can't specify an {target} bufsize if the {target} target is a pipe or TTY" - - if in_no_buf and in_buf is not None: - invalid.append((("in", "in_bufsize"), err.format(target="in"))) - - if out_no_buf and out_buf is not None: - invalid.append((("out", "out_bufsize"), err.format(target="out"))) - - return invalid - - -class Command(object): - """ represents an un-run system program, like "ls" or "cd". because it - represents the program itself (and not a running instance of it), it should - hold very little state. in fact, the only state it does hold is baked - arguments. - - when a Command object is called, the result that is returned is a - RunningCommand object, which represents the Command put into an execution - state. """ - thread_local = threading.local() - - _call_args = { - "fg": False, # run command in foreground - - # run a command in the background. commands run in the background - # ignore SIGHUP and do not automatically exit when the parent process - # ends - "bg": False, - - # automatically report exceptions for background commands - "bg_exc": True, - - "with": False, # prepend the command to every command after it - "in": None, - "out": None, # redirect STDOUT - "err": None, # redirect STDERR - "err_to_out": None, # redirect STDERR to STDOUT - - # stdin buffer size - # 1 for line, 0 for unbuffered, any other number for that amount - "in_bufsize": 0, - # stdout buffer size, same values as above - "out_bufsize": 1, - "err_bufsize": 1, - - # this is how big the output buffers will be for stdout and stderr. - # this is essentially how much output they will store from the process. - # we use a deque, so if it overflows past this amount, the first items - # get pushed off as each new item gets added. - # - # NOTICE - # this is not a *BYTE* size, this is a *CHUNK* size...meaning, that if - # you're buffering out/err at 1024 bytes, the internal buffer size will - # be "internal_bufsize" CHUNKS of 1024 bytes - "internal_bufsize": 3 * 1024 ** 2, - - "env": None, - "piped": None, - "iter": None, - "iter_noblock": None, - "ok_code": 0, - "cwd": None, - - # the separator delimiting between a long-argument's name and its value - # setting this to None will cause name and value to be two separate - # arguments, like for short options - # for example, --arg=derp, '=' is the long_sep - "long_sep": "=", - - # the prefix used for long arguments - "long_prefix": "--", - - # this is for programs that expect their input to be from a terminal. - # ssh is one of those programs - "tty_in": False, - "tty_out": True, - - "encoding": DEFAULT_ENCODING, - "decode_errors": "strict", - - # how long the process should run before it is auto-killed - "timeout": None, - "timeout_signal": signal.SIGKILL, - - # TODO write some docs on "long-running processes" - # these control whether or not stdout/err will get aggregated together - # as the process runs. this has memory usage implications, so sometimes - # with long-running processes with a lot of data, it makes sense to - # set these to true - "no_out": False, - "no_err": False, - "no_pipe": False, - - # if any redirection is used for stdout or stderr, internal buffering - # of that data is not stored. this forces it to be stored, as if - # the output is being T'd to both the redirected destination and our - # internal buffers - "tee": None, - - # will be called when a process terminates regardless of exception - "done": None, - - # a tuple (rows, columns) of the desired size of both the stdout and - # stdin ttys, if ttys are being used - "tty_size": (20, 80), - - # whether or not our exceptions should be truncated - "truncate_exc": True, - - # a function to call after the child forks but before the process execs - "preexec_fn": None, - - # UID to set after forking. Requires root privileges. Not supported on - # Windows. - "uid": None, - - # put the forked process in its own process session? - "new_session": True, - - # pre-process args passed into __call__. only really useful when used - # in .bake() - "arg_preprocess": None, - - # a callable that produces a log message from an argument tuple of the - # command and the args - "log_msg": None, - } - - # this is a collection of validators to make sure the special kwargs make - # sense - _kwarg_validators = ( - (("fg", "bg"), "Command can't be run in the foreground and background"), - (("fg", "err_to_out"), "Can't redirect STDERR in foreground mode"), - (("err", "err_to_out"), "Stderr is already being redirected"), - (("piped", "iter"), "You cannot iterate when this command is being piped"), - (("piped", "no_pipe"), "Using a pipe doesn't make sense if you've \ -disabled the pipe"), - (("no_out", "iter"), "You cannot iterate over output if there is no \ -output"), - tty_in_validator, - bufsize_validator, - ) - - - def __init__(self, path, search_paths=None): - found = which(path, search_paths) - - self._path = encode_to_py3bytes_or_py2str("") - - # is the command baked (aka, partially applied)? - self._partial = False - self._partial_baked_args = [] - self._partial_call_args = {} - - # bugfix for functools.wraps. issue #121 - self.__name__ = str(self) - - if not found: - raise CommandNotFound(path) - - # the reason why we set the values early in the constructor, and again - # here, is for people who have tools that inspect the stack on - # exception. if CommandNotFound is raised, we need self._path and the - # other attributes to be set correctly, so repr() works when they're - # inspecting the stack. issue #304 - self._path = encode_to_py3bytes_or_py2str(found) - self.__name__ = str(self) - - - def __getattribute__(self, name): - # convenience - getattr = partial(object.__getattribute__, self) - val = None - - if name.startswith("_"): - val = getattr(name) - - elif name == "bake": - val = getattr("bake") - - # here we have a way of getting past shadowed subcommands. for example, - # if "git bake" was a thing, we wouldn't be able to do `git.bake()` - # because `.bake()` is already a method. so we allow `git.bake_()` - elif name.endswith("_"): - name = name[:-1] - - if val is None: - val = getattr("bake")(name) - - return val - - - @staticmethod - def _extract_call_args(kwargs): - """ takes kwargs that were passed to a command's __call__ and extracts - out the special keyword arguments, we return a tuple of special keyword - args, and kwargs that will go to the execd command """ - - kwargs = kwargs.copy() - call_args = {} - for parg, default in Command._call_args.items(): - key = "_" + parg - - if key in kwargs: - call_args[parg] = kwargs[key] - del kwargs[key] - - invalid_kwargs = special_kwarg_validator(call_args, - Command._kwarg_validators) - - if invalid_kwargs: - exc_msg = [] - for args, error_msg in invalid_kwargs: - exc_msg.append(" %r: %s" % (args, error_msg)) - exc_msg = "\n".join(exc_msg) - raise TypeError("Invalid special arguments:\n\n%s\n" % exc_msg) - - return call_args, kwargs - - - # TODO needs documentation - def bake(self, *args, **kwargs): - fn = type(self)(self._path) - fn._partial = True - - call_args, kwargs = self._extract_call_args(kwargs) - - pruned_call_args = call_args - for k, v in Command._call_args.items(): - try: - if pruned_call_args[k] == v: - del pruned_call_args[k] - except KeyError: - continue - - fn._partial_call_args.update(self._partial_call_args) - fn._partial_call_args.update(pruned_call_args) - fn._partial_baked_args.extend(self._partial_baked_args) - sep = pruned_call_args.get("long_sep", self._call_args["long_sep"]) - prefix = pruned_call_args.get("long_prefix", - self._call_args["long_prefix"]) - fn._partial_baked_args.extend(compile_args(args, kwargs, sep, prefix)) - return fn - - def __str__(self): - """ in python3, should return unicode. in python2, should return a - string of bytes """ - if IS_PY3: - return self.__unicode__() - else: - return self.__unicode__().encode(DEFAULT_ENCODING) - - - def __eq__(self, other): - return str(self) == str(other) - - __hash__ = None # Avoid DeprecationWarning in Python < 3 - - - def __repr__(self): - """ in python3, should return unicode. in python2, should return a - string of bytes """ - return "" % str(self) - - - def __unicode__(self): - """ a magic method defined for python2. calling unicode() on a - self will call this """ - baked_args = " ".join(item.decode(DEFAULT_ENCODING) for item in self._partial_baked_args) - if baked_args: - baked_args = " " + baked_args - return self._path.decode(DEFAULT_ENCODING) + baked_args - - def __enter__(self): - self(_with=True) - - def __exit__(self, typ, value, traceback): - get_prepend_stack().pop() - - - def __call__(self, *args, **kwargs): - - kwargs = kwargs.copy() - args = list(args) - - # this will hold our final command, including arguments, that will be - # execd - cmd = [] - - # this will hold a complete mapping of all our special keyword arguments - # and their values - call_args = Command._call_args.copy() - - # aggregate any 'with' contexts - for prepend in get_prepend_stack(): - pcall_args = prepend.call_args.copy() - # don't pass the 'with' call arg - pcall_args.pop("with", None) - - call_args.update(pcall_args) - cmd.extend(prepend.cmd) - - cmd.append(self._path) - - # do we have an argument pre-processor? if so, run it. we need to do - # this early, so that args, kwargs are accurate - preprocessor = self._partial_call_args.get("arg_preprocess", None) - if preprocessor: - args, kwargs = preprocessor(args, kwargs) - - # here we extract the special kwargs and override any - # special kwargs from the possibly baked command - extracted_call_args, kwargs = self._extract_call_args(kwargs) - - call_args.update(self._partial_call_args) - call_args.update(extracted_call_args) - - - # handle a None. this is added back only to not break the api in the - # 1.* version. TODO remove this in 2.0, as "ok_code", if specified, - # should always be a definitive value or list of values, and None is - # ambiguous - if call_args["ok_code"] is None: - call_args["ok_code"] = 0 - - if not getattr(call_args["ok_code"], "__iter__", None): - call_args["ok_code"] = [call_args["ok_code"]] - - - # check if we're piping via composition - stdin = call_args["in"] - if args: - first_arg = args.pop(0) - if isinstance(first_arg, RunningCommand): - if first_arg.call_args["piped"]: - stdin = first_arg.process - else: - stdin = first_arg.process._pipe_queue - - else: - args.insert(0, first_arg) - - processed_args = compile_args(args, kwargs, call_args["long_sep"], - call_args["long_prefix"]) - - # makes sure our arguments are broken up correctly - split_args = self._partial_baked_args + processed_args - - final_args = split_args - - cmd.extend(final_args) - - # if we're running in foreground mode, we need to completely bypass - # launching a RunningCommand and OProc and just do a spawn - if call_args["fg"]: - if call_args["env"] is None: - launch = lambda: os.spawnv(os.P_WAIT, cmd[0], cmd) - else: - launch = lambda: os.spawnve(os.P_WAIT, cmd[0], cmd, call_args["env"]) - - exit_code = launch() - exc_class = get_exc_exit_code_would_raise(exit_code, call_args["ok_code"]) - if exc_class: - if IS_PY3: - ran = " ".join([arg.decode(DEFAULT_ENCODING, "ignore") for arg in cmd]) - else: - ran = " ".join(cmd) - exc = exc_class(ran, b"", b"", call_args["truncate_exc"]) - raise exc - return None - - - # stdout redirection - stdout = call_args["out"] - if output_redirect_is_filename(stdout): - stdout = open(str(stdout), "wb") - - # stderr redirection - stderr = call_args["err"] - if output_redirect_is_filename(stderr): - stderr = open(str(stderr), "wb") - - return RunningCommand(cmd, call_args, stdin, stdout, stderr) - - -def compile_args(args, kwargs, sep, prefix): - """ takes args and kwargs, as they were passed into the command instance - being executed with __call__, and compose them into a flat list that - will eventually be fed into exec. example: - - with this call: - - sh.ls("-l", "/tmp", color="never") - - this function receives - - args = ['-l', '/tmp'] - kwargs = {'color': 'never'} - - and produces - - ['-l', '/tmp', '--color=never'] - - """ - processed_args = [] - encode = encode_to_py3bytes_or_py2str - - # aggregate positional args - for arg in args: - if isinstance(arg, (list, tuple)): - if isinstance(arg, GlobResults) and not arg: - arg = [arg.path] - - for sub_arg in arg: - processed_args.append(encode(sub_arg)) - elif isinstance(arg, dict): - processed_args += aggregate_keywords(arg, sep, prefix, raw=True) - else: - processed_args.append(encode(arg)) - - # aggregate the keyword arguments - processed_args += aggregate_keywords(kwargs, sep, prefix) - - return processed_args - - -def aggregate_keywords(keywords, sep, prefix, raw=False): - """ take our keyword arguments, and a separator, and compose the list of - flat long (and short) arguments. example - - {'color': 'never', 't': True, 'something': True} with sep '=' - - becomes - - ['--color=never', '-t', '--something'] - - the `raw` argument indicates whether or not we should leave the argument - name alone, or whether we should replace "_" with "-". if we pass in a - dictionary, like this: - - sh.command({"some_option": 12}) - - then `raw` gets set to True, because we want to leave the key as-is, to - produce: - - ['--some_option=12'] - - but if we just use a command's kwargs, `raw` is False, which means this: - - sh.command(some_option=12) - - becomes: - - ['--some-option=12'] - - eessentially, using kwargs is a convenience, but it lacks the ability to - put a '-' in the name, so we do the replacement of '_' to '-' for you. - but when you really don't want that to happen, you should use a - dictionary instead with the exact names you want - """ - - processed = [] - encode = encode_to_py3bytes_or_py2str - - for k, v in keywords.items(): - # we're passing a short arg as a kwarg, example: - # cut(d="\t") - if len(k) == 1: - if v is not False: - processed.append(encode("-" + k)) - if v is not True: - processed.append(encode(v)) - - # we're doing a long arg - else: - if not raw: - k = k.replace("_", "-") - - if v is True: - processed.append(encode("--" + k)) - elif v is False: - pass - elif sep is None or sep == " ": - processed.append(encode(prefix + k)) - processed.append(encode(v)) - else: - arg = encode("%s%s%s%s" % (prefix, k, sep, v)) - processed.append(arg) - - return processed - - -def _start_daemon_thread(fn, name, exc_queue, *args): - def wrap(*args, **kwargs): - try: - fn(*args, **kwargs) - except Exception as e: - exc_queue.put(e) - raise - - thrd = threading.Thread(target=wrap, name=name, args=args) - thrd.daemon = True - thrd.start() - return thrd - - -def setwinsize(fd, rows_cols): - """ set the terminal size of a tty file descriptor. borrowed logic - from pexpect.py """ - rows, cols = rows_cols - TIOCSWINSZ = getattr(termios, 'TIOCSWINSZ', -2146929561) - - s = struct.pack('HHHH', rows, cols, 0, 0) - fcntl.ioctl(fd, TIOCSWINSZ, s) - -def construct_streamreader_callback(process, handler): - """ here we're constructing a closure for our streamreader callback. this - is used in the case that we pass a callback into _out or _err, meaning we - want to our callback to handle each bit of output - - we construct the closure based on how many arguments it takes. the reason - for this is to make it as easy as possible for people to use, without - limiting them. a new user will assume the callback takes 1 argument (the - data). as they get more advanced, they may want to terminate the process, - or pass some stdin back, and will realize that they can pass a callback of - more args """ - - - # implied arg refers to the "self" that methods will pass in. we need to - # account for this implied arg when figuring out what function the user - # passed in based on number of args - implied_arg = 0 - - partial_args = 0 - handler_to_inspect = handler - - if isinstance(handler, partial): - partial_args = len(handler.args) - handler_to_inspect = handler.func - - if inspect.ismethod(handler_to_inspect): - implied_arg = 1 - num_args = get_num_args(handler_to_inspect) - - else: - if inspect.isfunction(handler_to_inspect): - num_args = get_num_args(handler_to_inspect) - - # is an object instance with __call__ method - else: - implied_arg = 1 - num_args = get_num_args(handler_to_inspect.__call__) - - - net_args = num_args - implied_arg - partial_args - - handler_args = () - - # just the chunk - if net_args == 1: - handler_args = () - - # chunk, stdin - if net_args == 2: - handler_args = (process.stdin,) - - # chunk, stdin, process - elif net_args == 3: - # notice we're only storing a weakref, to prevent cyclic references - # (where the process holds a streamreader, and a streamreader holds a - # handler-closure with a reference to the process - handler_args = (process.stdin, weakref.ref(process)) - - def fn(chunk): - # this is pretty ugly, but we're evaluating the process at call-time, - # because it's a weakref - args = handler_args - if len(args) == 2: - args = (handler_args[0], handler_args[1]()) - return handler(chunk, *args) - - return fn - - -def get_exc_exit_code_would_raise(exit_code, ok_codes): - exc = None - success = exit_code in ok_codes - bad_sig = -exit_code in SIGNALS_THAT_SHOULD_THROW_EXCEPTION - - if not success or bad_sig: - exc = get_rc_exc(exit_code) - return exc - - -def handle_process_exit_code(exit_code): - """ this should only ever be called once for each child process """ - # if we exited from a signal, let our exit code reflect that - if os.WIFSIGNALED(exit_code): - exit_code = -os.WTERMSIG(exit_code) - # otherwise just give us a normal exit code - elif os.WIFEXITED(exit_code): - exit_code = os.WEXITSTATUS(exit_code) - else: - raise RuntimeError("Unknown child exit status!") - - return exit_code - - -def no_interrupt(syscall, *args, **kwargs): - """ a helper for making system calls immune to EINTR """ - ret = None - - while True: - try: - ret = syscall(*args, **kwargs) - except OSError as e: - if e.errno == errno.EINTR: - continue - else: - raise - else: - break - - return ret - - -class OProc(object): - """ this class is instantiated by RunningCommand for a command to be exec'd. - it handles all the nasty business involved with correctly setting up the - input/output to the child process. it gets its name for subprocess.Popen - (process open) but we're calling ours OProc (open process) """ - - _default_window_size = (24, 80) - - # used in redirecting - STDOUT = -1 - STDERR = -2 - - def __init__(self, command, parent_log, cmd, stdin, stdout, stderr, - call_args, pipe, process_assign_lock): - """ - cmd is the full string that will be exec'd. it includes the program - name and all its arguments - - stdin, stdout, stderr are what the child will use for standard - input/output/err - - call_args is a mapping of all the special keyword arguments to apply - to the child process - """ - self.command = command - self.call_args = call_args - - # convenience - ca = self.call_args - - if ca["uid"] is not None: - if os.getuid() != 0: - raise RuntimeError("UID setting requires root privileges") - - target_uid = ca["uid"] - - pwrec = pwd.getpwuid(ca["uid"]) - target_gid = pwrec.pw_gid - - # I had issues with getting 'Input/Output error reading stdin' from dd, - # until I set _tty_out=False - if ca["piped"]: - ca["tty_out"] = False - - self._stdin_process = None - - - # if the objects that we are passing to the OProc happen to be a - # file-like object that is a tty, for example `sys.stdin`, then, later - # on in this constructor, we're going to skip out on setting up pipes - # and pseudoterminals for those endpoints - stdin_is_tty_or_pipe = ob_is_tty(stdin) or ob_is_pipe(stdin) - stdout_is_tty_or_pipe = ob_is_tty(stdout) or ob_is_pipe(stdout) - stderr_is_tty_or_pipe = ob_is_tty(stderr) or ob_is_pipe(stderr) - - # if we're passing in a custom stdout/out/err value, we obviously have - # to force not using single_tty - custom_in_out_err = stdin or stdout or stderr - - single_tty = (ca["tty_in"] and ca["tty_out"]) and not custom_in_out_err - - # this logic is a little convoluted, but basically this top-level - # if/else is for consolidating input and output TTYs into a single - # TTY. this is the only way some secure programs like ssh will - # output correctly (is if stdout and stdin are both the same TTY) - if single_tty: - self._stdin_read_fd, self._stdin_write_fd = pty.openpty() - - self._stdout_read_fd = os.dup(self._stdin_read_fd) - self._stdout_write_fd = os.dup(self._stdin_write_fd) - - self._stderr_read_fd = os.dup(self._stdin_read_fd) - self._stderr_write_fd = os.dup(self._stdin_write_fd) - - # do not consolidate stdin and stdout. this is the most common use- - # case - else: - # this check here is because we may be doing piping and so our stdin - # might be an instance of OProc - if isinstance(stdin, OProc) and stdin.call_args["piped"]: - self._stdin_write_fd = stdin._pipe_fd - self._stdin_read_fd = None - self._stdin_process = stdin - - elif stdin_is_tty_or_pipe: - self._stdin_write_fd = os.dup(get_fileno(stdin)) - self._stdin_read_fd = None - - elif ca["tty_in"]: - self._stdin_read_fd, self._stdin_write_fd = pty.openpty() - - # tty_in=False is the default - else: - self._stdin_write_fd, self._stdin_read_fd = os.pipe() - - - if stdout_is_tty_or_pipe: - self._stdout_write_fd = os.dup(get_fileno(stdout)) - self._stdout_read_fd = None - - # tty_out=True is the default - elif ca["tty_out"]: - self._stdout_read_fd, self._stdout_write_fd = pty.openpty() - - else: - self._stdout_read_fd, self._stdout_write_fd = os.pipe() - - # unless STDERR is going to STDOUT, it ALWAYS needs to be a pipe, - # and never a PTY. the reason for this is not totally clear to me, - # but it has to do with the fact that if STDERR isn't set as the - # CTTY (because STDOUT is), the STDERR buffer won't always flush - # by the time the process exits, and the data will be lost. - # i've only seen this on OSX. - if stderr is OProc.STDOUT: - self._stderr_read_fd = os.dup(self._stdout_read_fd) - self._stderr_write_fd = os.dup(self._stdout_write_fd) - - elif stderr_is_tty_or_pipe: - self._stderr_write_fd = os.dup(get_fileno(stderr)) - self._stderr_read_fd = None - - else: - self._stderr_read_fd, self._stderr_write_fd = os.pipe() - - - piped = ca["piped"] - self._pipe_fd = None - if piped: - fd_to_use = self._stdout_read_fd - if piped == "err": - fd_to_use = self._stderr_read_fd - self._pipe_fd = os.dup(fd_to_use) - - - new_session = ca["new_session"] - needs_ctty = ca["tty_in"] and new_session - - self.ctty = None - if needs_ctty: - self.ctty = os.ttyname(self._stdin_write_fd) - - # this is a hack, but what we're doing here is intentionally throwing an - # OSError exception if our child processes's directory doesn't exist, - # but we're doing it BEFORE we fork. the reason for before the fork is - # error handling. i'm currently too lazy to implement what - # subprocess.py did and set up a error pipe to handle exceptions that - # happen in the child between fork and exec. it has only been seen in - # the wild for a missing cwd, so we'll handle it here. - cwd = ca["cwd"] - if cwd is not None and not os.path.exists(cwd): - os.chdir(cwd) - - gc_enabled = gc.isenabled() - if gc_enabled: - gc.disable() - - # for synchronizing - session_pipe_read, session_pipe_write = os.pipe() - exc_pipe_read, exc_pipe_write = os.pipe() - - # this pipe is for synchronzing with the child that the parent has - # closed its in/out/err fds. this is a bug on OSX (but not linux), - # where we can lose output sometimes, due to a race, if we do - # os.close(self._stdout_write_fd) in the parent after the child starts - # writing. - if IS_OSX: - close_pipe_read, close_pipe_write = os.pipe() - - - # session id, group id, process id - self.sid = None - self.pgid = None - self.pid = os.fork() - - # child - if self.pid == 0: # pragma: no cover - if IS_OSX: - os.read(close_pipe_read, 1) - os.close(close_pipe_read) - os.close(close_pipe_write) - - try: - # ignoring SIGHUP lets us persist even after the parent process - # exits. only ignore if we're backgrounded - if ca["bg"] is True: - signal.signal(signal.SIGHUP, signal.SIG_IGN) - - # put our forked process in a new session? this will relinquish - # any control of our inherited CTTY and also make our parent - # process init - if new_session: - os.setsid() - # if we're not going in a new session, we should go in a new - # process group. this way, our process, and any children it - # spawns, are alone, contained entirely in one group. if we - # didn't do this, and didn't use a new session, then our exec'd - # process *could* exist in the same group as our python process, - # depending on how we launch the process (from a shell, or some - # other way) - else: - os.setpgrp() - - sid = os.getsid(0) - pgid = os.getpgid(0) - payload = ("%d,%d" % (sid, pgid)).encode(DEFAULT_ENCODING) - os.write(session_pipe_write, payload) - - if ca["tty_out"] and not stdout_is_tty_or_pipe and not single_tty: - # set raw mode, so there isn't any weird translation of - # newlines to \r\n and other oddities. we're not outputting - # to a terminal anyways - # - # we HAVE to do this here, and not in the parent process, - # because we have to guarantee that this is set before the - # child process is run, and we can't do it twice. - tty.setraw(self._stdout_write_fd) - - - # if the parent-side fd for stdin exists, close it. the case - # where it may not exist is if we're using piping - if self._stdin_read_fd: - os.close(self._stdin_read_fd) - - if self._stdout_read_fd: - os.close(self._stdout_read_fd) - - if self._stderr_read_fd: - os.close(self._stderr_read_fd) - - os.close(session_pipe_read) - os.close(exc_pipe_read) - - if cwd: - os.chdir(cwd) - - os.dup2(self._stdin_write_fd, 0) - os.dup2(self._stdout_write_fd, 1) - os.dup2(self._stderr_write_fd, 2) - - - # set our controlling terminal, but only if we're using a tty - # for stdin. it doesn't make sense to have a ctty otherwise - if needs_ctty: - tmp_fd = os.open(os.ttyname(0), os.O_RDWR) - os.close(tmp_fd) - - if ca["tty_out"] and not stdout_is_tty_or_pipe: - setwinsize(1, ca["tty_size"]) - - if ca["uid"] is not None: - os.setgid(target_gid) - os.setuid(target_uid) - - preexec_fn = ca["preexec_fn"] - if callable(preexec_fn): - preexec_fn() - - - # don't inherit file descriptors - max_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0] - os.closerange(3, max_fd) - - # actually execute the process - if ca["env"] is None: - os.execv(cmd[0], cmd) - else: - os.execve(cmd[0], cmd, ca["env"]) - - # we must ensure that we carefully exit the child process on - # exception, otherwise the parent process code will be executed - # twice on exception https://github.com/amoffat/sh/issues/202 - # - # if your parent process experiences an exit code 255, it is most - # likely that an exception occurred between the fork of the child - # and the exec. this should be reported. - except: - # some helpful debugging - try: - tb = traceback.format_exc().encode("utf8", "ignore") - os.write(exc_pipe_write, tb) - - finally: - os._exit(255) - - # parent - else: - if gc_enabled: - gc.enable() - - os.close(self._stdin_write_fd) - os.close(self._stdout_write_fd) - os.close(self._stderr_write_fd) - - # tell our child process that we've closed our write_fds, so it is - # ok to proceed towards exec. see the comment where this pipe is - # opened, for why this is necessary - if IS_OSX: - os.close(close_pipe_read) - os.write(close_pipe_write, str(1).encode(DEFAULT_ENCODING)) - os.close(close_pipe_write) - - os.close(exc_pipe_write) - fork_exc = os.read(exc_pipe_read, 1024**2) - os.close(exc_pipe_read) - if fork_exc: - fork_exc = fork_exc.decode(DEFAULT_ENCODING) - raise ForkException(fork_exc) - - os.close(session_pipe_write) - sid, pgid = os.read(session_pipe_read, - 1024).decode(DEFAULT_ENCODING).split(",") - os.close(session_pipe_read) - self.sid = int(sid) - self.pgid = int(pgid) - - # used to determine what exception to raise. if our process was - # killed via a timeout counter, we'll raise something different than - # a SIGKILL exception - self.timed_out = False - - self.started = time.time() - self.cmd = cmd - - # exit code should only be manipulated from within self._wait_lock - # to prevent race conditions - self.exit_code = None - - self.stdin = stdin or Queue() - - # _pipe_queue is used internally to hand off stdout from one process - # to another. by default, all stdout from a process gets dumped - # into this pipe queue, to be consumed in real time (hence the - # thread-safe Queue), or at a potentially later time - self._pipe_queue = Queue() - - # this is used to prevent a race condition when we're waiting for - # a process to end, and the OProc's internal threads are also checking - # for the processes's end - self._wait_lock = threading.Lock() - - # these are for aggregating the stdout and stderr. we use a deque - # because we don't want to overflow - self._stdout = deque(maxlen=ca["internal_bufsize"]) - self._stderr = deque(maxlen=ca["internal_bufsize"]) - - if ca["tty_in"] and not stdin_is_tty_or_pipe: - setwinsize(self._stdin_read_fd, ca["tty_size"]) - - - self.log = parent_log.get_child("process", repr(self)) - - - self.log.debug("started process") - - # disable echoing, but only if it's a tty that we created ourselves - if ca["tty_in"] and not stdin_is_tty_or_pipe: - attr = termios.tcgetattr(self._stdin_read_fd) - attr[3] &= ~termios.ECHO - termios.tcsetattr(self._stdin_read_fd, termios.TCSANOW, attr) - - # we're only going to create a stdin thread iff we have potential - # for stdin to come in. this would be through a stdout callback or - # through an object we've passed in for stdin - potentially_has_input = callable(stdout) or stdin - - # this represents the connection from a Queue object (or whatever - # we're using to feed STDIN) to the process's STDIN fd - self._stdin_stream = None - if self._stdin_read_fd and potentially_has_input: - log = self.log.get_child("streamwriter", "stdin") - self._stdin_stream = StreamWriter(log, self._stdin_read_fd, - self.stdin, ca["in_bufsize"], ca["encoding"], - ca["tty_in"]) - - stdout_pipe = None - if pipe is OProc.STDOUT and not ca["no_pipe"]: - stdout_pipe = self._pipe_queue - - - # this represents the connection from a process's STDOUT fd to - # wherever it has to go, sometimes a pipe Queue (that we will use - # to pipe data to other processes), and also an internal deque - # that we use to aggregate all the output - save_stdout = not ca["no_out"] and \ - (ca["tee"] in (True, "out") or stdout is None) - - - pipe_out = ca["piped"] in ("out", True) - pipe_err = ca["piped"] in ("err",) - - # if we're piping directly into another process's filedescriptor, we - # bypass reading from the stdout stream altogether, because we've - # already hooked up this processes's stdout fd to the other - # processes's stdin fd - self._stdout_stream = None - if not pipe_out and self._stdout_read_fd: - if callable(stdout): - stdout = construct_streamreader_callback(self, stdout) - self._stdout_stream = \ - StreamReader( - self.log.get_child("streamreader", "stdout"), - self._stdout_read_fd, stdout, self._stdout, - ca["out_bufsize"], ca["encoding"], - ca["decode_errors"], stdout_pipe, - save_data=save_stdout) - - elif self._stdout_read_fd: - os.close(self._stdout_read_fd) - - - # if stderr is going to one place (because it's grouped with stdout, - # or we're dealing with a single tty), then we don't actually need a - # stream reader for stderr, because we've already set one up for - # stdout above - self._stderr_stream = None - if stderr is not OProc.STDOUT and not single_tty and not pipe_err \ - and self._stderr_read_fd: - - stderr_pipe = None - if pipe is OProc.STDERR and not ca["no_pipe"]: - stderr_pipe = self._pipe_queue - - save_stderr = not ca["no_err"] and \ - (ca["tee"] in ("err",) or stderr is None) - - if callable(stderr): - stderr = construct_streamreader_callback(self, stderr) - - self._stderr_stream = StreamReader(Logger("streamreader"), - self._stderr_read_fd, stderr, self._stderr, - ca["err_bufsize"], ca["encoding"], ca["decode_errors"], - stderr_pipe, save_data=save_stderr) - - elif self._stderr_read_fd: - os.close(self._stderr_read_fd) - - - def timeout_fn(): - self.timed_out = True - self.signal(ca["timeout_signal"]) - - - self._timeout_event = None - self._timeout_timer = None - if ca["timeout"]: - self._timeout_event = threading.Event() - self._timeout_timer = threading.Timer(ca["timeout"], - self._timeout_event.set) - self._timeout_timer.start() - - # this is for cases where we know that the RunningCommand that was - # launched was not .wait()ed on to complete. in those unique cases, - # we allow the thread that processes output to report exceptions in - # that thread. it's important that we only allow reporting of the - # exception, and nothing else (like the additional stuff that - # RunningCommand.wait() does), because we want the exception to be - # re-raised in the future, if we DO call .wait() - handle_exit_code = None - if not self.command._spawned_and_waited and ca["bg_exc"]: - def fn(exit_code): - with process_assign_lock: - return self.command.handle_command_exit_code(exit_code) - handle_exit_code = fn - - self._quit_threads = threading.Event() - - thread_name = "background thread for pid %d" % self.pid - self._bg_thread_exc_queue = Queue(1) - self._background_thread = _start_daemon_thread(background_thread, - thread_name, self._bg_thread_exc_queue, timeout_fn, - self._timeout_event, handle_exit_code, self.is_alive, - self._quit_threads) - - - # start the main io threads. stdin thread is not needed if we are - # connecting from another process's stdout pipe - self._input_thread = None - self._input_thread_exc_queue = Queue(1) - if self._stdin_stream: - close_before_term = not needs_ctty - thread_name = "STDIN thread for pid %d" % self.pid - self._input_thread = _start_daemon_thread(input_thread, - thread_name, self._input_thread_exc_queue, self.log, - self._stdin_stream, self.is_alive, self._quit_threads, - close_before_term) - - - # this event is for cases where the subprocess that we launch - # launches its OWN subprocess and dups the stdout/stderr fds to that - # new subprocess. in that case, stdout and stderr will never EOF, - # so our output_thread will never finish and will hang. this event - # prevents that hanging - self._stop_output_event = threading.Event() - - self._output_thread_exc_queue = Queue(1) - thread_name = "STDOUT/ERR thread for pid %d" % self.pid - self._output_thread = _start_daemon_thread(output_thread, - thread_name, self._output_thread_exc_queue, self.log, - self._stdout_stream, self._stderr_stream, - self._timeout_event, self.is_alive, self._quit_threads, - self._stop_output_event) - - - def __repr__(self): - return "" % (self.pid, self.cmd[:500]) - - - # these next 3 properties are primary for tests - @property - def output_thread_exc(self): - exc = None - try: - exc = self._output_thread_exc_queue.get(False) - except Empty: - pass - return exc - - @property - def input_thread_exc(self): - exc = None - try: - exc = self._input_thread_exc_queue.get(False) - except Empty: - pass - return exc - - @property - def bg_thread_exc(self): - exc = None - try: - exc = self._bg_thread_exc_queue.get(False) - except Empty: - pass - return exc - - - def change_in_bufsize(self, buf): - self._stdin_stream.stream_bufferer.change_buffering(buf) - - def change_out_bufsize(self, buf): - self._stdout_stream.stream_bufferer.change_buffering(buf) - - def change_err_bufsize(self, buf): - self._stderr_stream.stream_bufferer.change_buffering(buf) - - - - @property - def stdout(self): - return "".encode(self.call_args["encoding"]).join(self._stdout) - - @property - def stderr(self): - return "".encode(self.call_args["encoding"]).join(self._stderr) - - def get_pgid(self): - """ return the CURRENT group id of the process. this differs from - self.pgid in that this refects the current state of the process, where - self.pgid is the group id at launch """ - return os.getpgid(self.pid) - - def get_sid(self): - """ return the CURRENT session id of the process. this differs from - self.sid in that this refects the current state of the process, where - self.sid is the session id at launch """ - return os.getsid(self.pid) - - def signal_group(self, sig): - self.log.debug("sending signal %d to group", sig) - os.killpg(self.get_pgid(), sig) - - def signal(self, sig): - self.log.debug("sending signal %d", sig) - os.kill(self.pid, sig) - - def kill_group(self): - self.log.debug("killing group") - self.signal_group(signal.SIGKILL) - - def kill(self): - self.log.debug("killing") - self.signal(signal.SIGKILL) - - def terminate(self): - self.log.debug("terminating") - self.signal(signal.SIGTERM) - - - def is_alive(self): - """ polls if our child process has completed, without blocking. this - method has side-effects, such as setting our exit_code, if we happen to - see our child exit while this is running """ - - if self.exit_code is not None: - return False, self.exit_code - - # what we're doing here essentially is making sure that the main thread - # (or another thread), isn't calling .wait() on the process. because - # .wait() calls os.waitpid(self.pid, 0), we can't do an os.waitpid - # here...because if we did, and the process exited while in this - # thread, the main thread's os.waitpid(self.pid, 0) would raise OSError - # (because the process ended in another thread). - # - # so essentially what we're doing is, using this lock, checking if - # we're calling .wait(), and if we are, let .wait() get the exit code - # and handle the status, otherwise let us do it. - acquired = self._wait_lock.acquire(False) - if not acquired: - if self.exit_code is not None: - return False, self.exit_code - return True, self.exit_code - - try: - # WNOHANG is just that...we're calling waitpid without hanging... - # essentially polling the process. the return result is (0, 0) if - # there's no process status, so we check that pid == self.pid below - # in order to determine how to proceed - pid, exit_code = no_interrupt(os.waitpid, self.pid, os.WNOHANG) - if pid == self.pid: - self.exit_code = handle_process_exit_code(exit_code) - self._process_just_ended() - - return False, self.exit_code - - # no child process - except OSError: - return False, self.exit_code - else: - return True, self.exit_code - finally: - self._wait_lock.release() - - - def _process_just_ended(self): - if self._timeout_timer: - self._timeout_timer.cancel() - - done_callback = self.call_args["done"] - if done_callback: - success = self.exit_code in self.call_args["ok_code"] - done_callback(success, self.exit_code) - - # this can only be closed at the end of the process, because it might be - # the CTTY, and closing it prematurely will send a SIGHUP. we also - # don't want to close it if there's a self._stdin_stream, because that - # is in charge of closing it also - if self._stdin_read_fd and not self._stdin_stream: - os.close(self._stdin_read_fd) - - - def wait(self): - """ waits for the process to complete, handles the exit code """ - - self.log.debug("acquiring wait lock to wait for completion") - # using the lock in a with-context blocks, which is what we want if - # we're running wait() - with self._wait_lock: - self.log.debug("got wait lock") - witnessed_end = False - - if self.exit_code is None: - self.log.debug("exit code not set, waiting on pid") - pid, exit_code = no_interrupt(os.waitpid, self.pid, 0) # blocks - self.exit_code = handle_process_exit_code(exit_code) - witnessed_end = True - - else: - self.log.debug("exit code already set (%d), no need to wait", - self.exit_code) - - self._quit_threads.set() - - # we may not have a thread for stdin, if the pipe has been connected - # via _piped="direct" - if self._input_thread: - self._input_thread.join() - - # wait, then signal to our output thread that the child process is - # done, and we should have finished reading all the stdout/stderr - # data that we can by now - timer = threading.Timer(2.0, self._stop_output_event.set) - timer.start() - - # wait for our stdout and stderr streamreaders to finish reading and - # aggregating the process output - self._output_thread.join() - timer.cancel() - - self._background_thread.join() - - if witnessed_end: - self._process_just_ended() - - return self.exit_code - - - -def input_thread(log, stdin, is_alive, quit, close_before_term): - """ this is run in a separate thread. it writes into our process's - stdin (a streamwriter) and waits the process to end AND everything that - can be written to be written """ - - done = False - closed = False - alive = True - writers = [stdin] - - while writers and alive: - _, to_write, _ = select.select([], writers, [], 1) - - if to_write: - log.debug("%r ready for more input", stdin) - done = stdin.write() - - if done: - writers = [] - if close_before_term: - stdin.close() - closed = True - - alive, _ = is_alive() - - while alive: - quit.wait(1) - alive, _ = is_alive() - - if not closed: - stdin.close() - - -def event_wait(ev, timeout=None): - triggered = ev.wait(timeout) - if IS_PY26: - triggered = ev.is_set() - return triggered - - -def background_thread(timeout_fn, timeout_event, handle_exit_code, is_alive, - quit): - """ handles the timeout logic """ - - # if there's a timeout event, loop - if timeout_event: - while not quit.is_set(): - timed_out = event_wait(timeout_event, 0.1) - if timed_out: - timeout_fn() - break - - # handle_exit_code will be a function ONLY if our command was NOT waited on - # as part of its spawning. in other words, it's probably a background - # command - # - # this reports the exit code exception in our thread. it's purely for the - # user's awareness, and cannot be caught or used in any way, so it's ok to - # suppress this during the tests - if handle_exit_code and not RUNNING_TESTS: # pragma: no cover - alive = True - while alive: - quit.wait(1) - alive, exit_code = is_alive() - - handle_exit_code(exit_code) - - -def output_thread(log, stdout, stderr, timeout_event, is_alive, quit, - stop_output_event): - """ this function is run in a separate thread. it reads from the - process's stdout stream (a streamreader), and waits for it to claim that - its done """ - - readers = [] - errors = [] - - if stdout is not None: - readers.append(stdout) - errors.append(stdout) - if stderr is not None: - readers.append(stderr) - errors.append(stderr) - - # this is our select loop for polling stdout or stderr that is ready to - # be read and processed. if one of those streamreaders indicate that it - # is done altogether being read from, we remove it from our list of - # things to poll. when no more things are left to poll, we leave this - # loop and clean up - while readers: - outputs, inputs, err = no_interrupt(select.select, readers, [], errors, 1) - - # stdout and stderr - for stream in outputs: - log.debug("%r ready to be read from", stream) - done = stream.read() - if done: - readers.remove(stream) - - # for some reason, we have to just ignore streams that have had an - # error. i'm not exactly sure why, but don't remove this until we - # figure that out, and create a test for it - for stream in err: - pass - - if timeout_event and timeout_event.is_set(): - break - - if stop_output_event.is_set(): - break - - # we need to wait until the process is guaranteed dead before closing our - # outputs, otherwise SIGPIPE - alive = True - while alive: - quit.wait(1) - alive, _ = is_alive() - - if stdout: - stdout.close() - - if stderr: - stderr.close() - - -class DoneReadingForever(Exception): pass -class NotYetReadyToRead(Exception): pass - - -def determine_how_to_read_input(input_obj): - """ given some kind of input object, return a function that knows how to - read chunks of that input object. - - each reader function should return a chunk and raise a DoneReadingForever - exception, or return None, when there's no more data to read - - NOTE: the function returned does not need to care much about the requested - buffering type (eg, unbuffered vs newline-buffered). the StreamBufferer - will take care of that. these functions just need to return a - reasonably-sized chunk of data. """ - - get_chunk = None - - if isinstance(input_obj, Queue): - log_msg = "queue" - get_chunk = get_queue_chunk_reader(input_obj) - - elif callable(input_obj): - log_msg = "callable" - get_chunk = get_callable_chunk_reader(input_obj) - - # also handles stringio - elif hasattr(input_obj, "read"): - log_msg = "file descriptor" - get_chunk = get_file_chunk_reader(input_obj) - - elif isinstance(input_obj, basestring): - log_msg = "string" - get_chunk = get_iter_string_reader(input_obj) - - elif isinstance(input_obj, bytes): - log_msg = "bytes" - get_chunk = get_iter_string_reader(input_obj) - - elif isinstance(input_obj, GeneratorType): - log_msg = "generator" - get_chunk = get_iter_chunk_reader(iter(input_obj)) - - else: - try: - it = iter(input_obj) - except TypeError: - raise Exception("unknown input object") - else: - log_msg = "general iterable" - get_chunk = get_iter_chunk_reader(it) - - return get_chunk, log_msg - - - -def get_queue_chunk_reader(stdin): - def fn(): - try: - chunk = stdin.get(True, 0.1) - except Empty: - raise NotYetReadyToRead - if chunk is None: - raise DoneReadingForever - return chunk - return fn - - -def get_callable_chunk_reader(stdin): - def fn(): - try: - data = stdin() - except DoneReadingForever: - raise - - if not data: - raise DoneReadingForever - - return data - - return fn - - -def get_iter_string_reader(stdin): - """ return an iterator that returns a chunk of a string every time it is - called. notice that even though bufsize_type might be line buffered, we're - not doing any line buffering here. that's because our StreamBufferer - handles all buffering. we just need to return a reasonable-sized chunk. """ - bufsize = 1024 - iter_str = (stdin[i:i + bufsize] for i in range(0, len(stdin), bufsize)) - return get_iter_chunk_reader(iter_str) - - -def get_iter_chunk_reader(stdin): - def fn(): - try: - if IS_PY3: - chunk = stdin.__next__() - else: - chunk = stdin.next() - return chunk - except StopIteration: - raise DoneReadingForever - return fn - -def get_file_chunk_reader(stdin): - bufsize = 1024 - - def fn(): - # python 3.* includes a fileno on stringios, but accessing it throws an - # exception. that exception is how we'll know we can't do a select on - # stdin - is_real_file = True - if IS_PY3: - try: - stdin.fileno() - except UnsupportedOperation: - is_real_file = False - - # this select is for files that may not yet be ready to read. we test - # for fileno because StringIO/BytesIO cannot be used in a select - if is_real_file and hasattr(stdin, "fileno"): - outputs, _, _ = select.select([stdin], [], [], 0.1) - if not outputs: - raise NotYetReadyToRead - - chunk = stdin.read(bufsize) - if not chunk: - raise DoneReadingForever - else: - return chunk - - return fn - - -def bufsize_type_to_bufsize(bf_type): - """ for a given bufsize type, return the actual bufsize we will read. - notice that although 1 means "newline-buffered", we're reading a chunk size - of 1024. this is because we have to read something. we let a - StreamBufferer instance handle splitting our chunk on newlines """ - - # newlines - if bf_type == 1: - bufsize = 1024 - # unbuffered - elif bf_type == 0: - bufsize = 1 - # or buffered by specific amount - else: - bufsize = bf_type - - return bufsize - - - -class StreamWriter(object): - """ StreamWriter reads from some input (the stdin param) and writes to a fd - (the stream param). the stdin may be a Queue, a callable, something with - the "read" method, a string, or an iterable """ - - def __init__(self, log, stream, stdin, bufsize_type, encoding, tty_in): - - self.stream = stream - self.stdin = stdin - - self.log = log - self.encoding = encoding - self.tty_in = tty_in - - self.stream_bufferer = StreamBufferer(bufsize_type, self.encoding) - self.get_chunk, log_msg = determine_how_to_read_input(stdin) - self.log.debug("parsed stdin as a %s", log_msg) - - - def fileno(self): - """ defining this allows us to do select.select on an instance of this - class """ - return self.stream - - - - def write(self): - """ attempt to get a chunk of data to write to our child process's - stdin, then write it. the return value answers the questions "are we - done writing forever?" """ - - # get_chunk may sometimes return bytes, and sometimes return strings - # because of the nature of the different types of STDIN objects we - # support - try: - chunk = self.get_chunk() - if chunk is None: - raise DoneReadingForever - - except DoneReadingForever: - self.log.debug("done reading") - - if self.tty_in: - # EOF time - try: - char = termios.tcgetattr(self.stream)[6][termios.VEOF] - except: - char = chr(4).encode() - - # normally, one EOF should be enough to signal to an program - # that is read()ing, to return 0 and be on your way. however, - # some programs are misbehaved, like python3.1 and python3.2. - # they don't stop reading sometimes after read() returns 0. - # this can be demonstrated with the following program: - # - # import sys - # sys.stdout.write(sys.stdin.read()) - # - # then type 'a' followed by ctrl-d 3 times. in python - # 2.6,2.7,3.3,3.4,3.5, it only takes 2 ctrl-d to terminate. - # however, in python 3.1 and 3.2, it takes all 3. - # - # so here we send an extra EOF along, just in case. i don't - # believe it can hurt anything - os.write(self.stream, char) - os.write(self.stream, char) - - return True - - except NotYetReadyToRead: - self.log.debug("received no data") - return False - - # if we're not bytes, make us bytes - if IS_PY3 and hasattr(chunk, "encode"): - chunk = chunk.encode(self.encoding) - - for proc_chunk in self.stream_bufferer.process(chunk): - self.log.debug("got chunk size %d: %r", len(proc_chunk), - proc_chunk[:30]) - - self.log.debug("writing chunk to process") - try: - os.write(self.stream, proc_chunk) - except OSError: - self.log.debug("OSError writing stdin chunk") - return True - - - def close(self): - self.log.debug("closing, but flushing first") - chunk = self.stream_bufferer.flush() - self.log.debug("got chunk size %d to flush: %r", len(chunk), chunk[:30]) - try: - if chunk: - os.write(self.stream, chunk) - - except OSError: - pass - - os.close(self.stream) - - -def determine_how_to_feed_output(handler, encoding, decode_errors): - if callable(handler): - process, finish = get_callback_chunk_consumer(handler, encoding, - decode_errors) - - # in py3, this is used for bytes - elif isinstance(handler, (cStringIO, iocStringIO)): - process, finish = get_cstringio_chunk_consumer(handler) - - # in py3, this is used for unicode - elif isinstance(handler, (StringIO, ioStringIO)): - process, finish = get_stringio_chunk_consumer(handler, encoding, - decode_errors) - - elif hasattr(handler, "write"): - process, finish = get_file_chunk_consumer(handler) - - else: - process = lambda chunk: False - finish = lambda: None - - return process, finish - - -def get_file_chunk_consumer(handler): - encode = lambda chunk: chunk - if getattr(handler, "encoding", None): - encode = lambda chunk: chunk.decode(handler.encoding) - - flush = lambda: None - if hasattr(handler, "flush"): - flush = handler.flush - - def process(chunk): - handler.write(encode(chunk)) - # we should flush on an fd. chunk is already the correctly-buffered - # size, so we don't need the fd buffering as well - flush() - return False - - def finish(): - flush() - - return process, finish - -def get_callback_chunk_consumer(handler, encoding, decode_errors): - def process(chunk): - # try to use the encoding first, if that doesn't work, send - # the bytes, because it might be binary - try: - chunk = chunk.decode(encoding, decode_errors) - except UnicodeDecodeError: - pass - return handler(chunk) - - def finish(): - pass - - return process, finish - -def get_cstringio_chunk_consumer(handler): - def process(chunk): - handler.write(chunk) - return False - - def finish(): - pass - - return process, finish - - -def get_stringio_chunk_consumer(handler, encoding, decode_errors): - def process(chunk): - handler.write(chunk.decode(encoding, decode_errors)) - return False - - def finish(): - pass - - return process, finish - - -class StreamReader(object): - """ reads from some output (the stream) and sends what it just read to the - handler. """ - def __init__(self, log, stream, handler, buffer, bufsize_type, encoding, - decode_errors, pipe_queue=None, save_data=True): - self.stream = stream - self.buffer = buffer - self.save_data = save_data - self.encoding = encoding - self.decode_errors = decode_errors - - self.pipe_queue = None - if pipe_queue: - self.pipe_queue = weakref.ref(pipe_queue) - - self.log = log - - self.stream_bufferer = StreamBufferer(bufsize_type, self.encoding, - self.decode_errors) - self.bufsize = bufsize_type_to_bufsize(bufsize_type) - - self.process_chunk, self.finish_chunk_processor = \ - determine_how_to_feed_output(handler, encoding, decode_errors) - - self.should_quit = False - - - def fileno(self): - """ defining this allows us to do select.select on an instance of this - class """ - return self.stream - - def close(self): - chunk = self.stream_bufferer.flush() - self.log.debug("got chunk size %d to flush: %r", len(chunk), chunk[:30]) - if chunk: - self.write_chunk(chunk) - - self.finish_chunk_processor() - - if self.pipe_queue and self.save_data: - self.pipe_queue().put(None) - - os.close(self.stream) - - - def write_chunk(self, chunk): - # in PY3, the chunk coming in will be bytes, so keep that in mind - - if not self.should_quit: - self.should_quit = self.process_chunk(chunk) - - - if self.save_data: - self.buffer.append(chunk) - - if self.pipe_queue: - self.log.debug("putting chunk onto pipe: %r", chunk[:30]) - self.pipe_queue().put(chunk) - - - def read(self): - # if we're PY3, we're reading bytes, otherwise we're reading - # str - try: - chunk = no_interrupt(os.read, self.stream, self.bufsize) - except OSError as e: - self.log.debug("got errno %d, done reading", e.errno) - return True - if not chunk: - self.log.debug("got no chunk, done reading") - return True - - self.log.debug("got chunk size %d: %r", len(chunk), chunk[:30]) - for chunk in self.stream_bufferer.process(chunk): - self.write_chunk(chunk) - - - - -class StreamBufferer(object): - """ this is used for feeding in chunks of stdout/stderr, and breaking it up - into chunks that will actually be put into the internal buffers. for - example, if you have two processes, one being piped to the other, and you - want that, first process to feed lines of data (instead of the chunks - however they come in), OProc will use an instance of this class to chop up - the data and feed it as lines to be sent down the pipe """ - - def __init__(self, buffer_type, encoding=DEFAULT_ENCODING, - decode_errors="strict"): - # 0 for unbuffered, 1 for line, everything else for that amount - self.type = buffer_type - self.buffer = [] - self.n_buffer_count = 0 - self.encoding = encoding - self.decode_errors = decode_errors - - # this is for if we change buffering types. if we change from line - # buffered to unbuffered, its very possible that our self.buffer list - # has data that was being saved up (while we searched for a newline). - # we need to use that up, so we don't lose it - self._use_up_buffer_first = False - - # the buffering lock is used because we might change the buffering - # types from a different thread. for example, if we have a stdout - # callback, we might use it to change the way stdin buffers. so we - # lock - self._buffering_lock = threading.RLock() - self.log = Logger("stream_bufferer") - - - def change_buffering(self, new_type): - # TODO, when we stop supporting 2.6, make this a with context - self.log.debug("acquiring buffering lock for changing buffering") - self._buffering_lock.acquire() - self.log.debug("got buffering lock for changing buffering") - try: - if new_type == 0: - self._use_up_buffer_first = True - - self.type = new_type - finally: - self._buffering_lock.release() - self.log.debug("released buffering lock for changing buffering") - - - def process(self, chunk): - # MAKE SURE THAT THE INPUT IS PY3 BYTES - # THE OUTPUT IS ALWAYS PY3 BYTES - - # TODO, when we stop supporting 2.6, make this a with context - self.log.debug("acquiring buffering lock to process chunk (buffering: %d)", self.type) - self._buffering_lock.acquire() - self.log.debug("got buffering lock to process chunk (buffering: %d)", self.type) - try: - # unbuffered - if self.type == 0: - if self._use_up_buffer_first: - self._use_up_buffer_first = False - to_write = self.buffer - self.buffer = [] - to_write.append(chunk) - return to_write - - return [chunk] - - # line buffered - elif self.type == 1: - total_to_write = [] - nl = "\n".encode(self.encoding) - while True: - newline = chunk.find(nl) - if newline == -1: - break - - chunk_to_write = chunk[:newline + 1] - if self.buffer: - chunk_to_write = b"".join(self.buffer) + chunk_to_write - - self.buffer = [] - self.n_buffer_count = 0 - - chunk = chunk[newline + 1:] - total_to_write.append(chunk_to_write) - - if chunk: - self.buffer.append(chunk) - self.n_buffer_count += len(chunk) - return total_to_write - - # N size buffered - else: - total_to_write = [] - while True: - overage = self.n_buffer_count + len(chunk) - self.type - if overage >= 0: - ret = "".encode(self.encoding).join(self.buffer) + chunk - chunk_to_write = ret[:self.type] - chunk = ret[self.type:] - total_to_write.append(chunk_to_write) - self.buffer = [] - self.n_buffer_count = 0 - else: - self.buffer.append(chunk) - self.n_buffer_count += len(chunk) - break - return total_to_write - finally: - self._buffering_lock.release() - self.log.debug("released buffering lock for processing chunk (buffering: %d)", self.type) - - - def flush(self): - self.log.debug("acquiring buffering lock for flushing buffer") - self._buffering_lock.acquire() - self.log.debug("got buffering lock for flushing buffer") - try: - ret = "".encode(self.encoding).join(self.buffer) - self.buffer = [] - return ret - finally: - self._buffering_lock.release() - self.log.debug("released buffering lock for flushing buffer") - - - -def with_lock(lock): - def wrapped(fn): - fn = contextmanager(fn) - @contextmanager - def wrapped2(*args, **kwargs): - with lock: - with fn(*args, **kwargs): - yield - return wrapped2 - return wrapped - - -@with_lock(PUSHD_LOCK) -def pushd(path): - """ pushd changes the actual working directory for the duration of the - context, unlike the _cwd arg this will work with other built-ins such as - sh.glob correctly """ - orig_path = os.getcwd() - os.chdir(path) - try: - yield - finally: - os.chdir(orig_path) - - -@contextmanager -def args(**kwargs): - """ allows us to temporarily override all the special keyword parameters in - a with context """ - - kwargs_str = ",".join(["%s=%r" % (k,v) for k,v in kwargs.items()]) - - raise DeprecationWarning(""" - -sh.args() has been deprecated because it was never thread safe. use the -following instead: - - sh2 = sh({kwargs}) - sh2.your_command() - -or - - sh2 = sh({kwargs}) - from sh2 import your_command - your_command() - -""".format(kwargs=kwargs_str)) - - - -class Environment(dict): - """ this allows lookups to names that aren't found in the global scope to be - searched for as a program name. for example, if "ls" isn't found in this - module's scope, we consider it a system program and try to find it. - - we use a dict instead of just a regular object as the base class because the - exec() statement used in the run_repl requires the "globals" argument to be a - dictionary """ - - - # this is a list of all of the names that the sh module exports that will - # not resolve to functions. we don't want to accidentally shadow real - # commands with functions/imports that we define in sh.py. for example, - # "import time" may override the time system program - whitelist = set([ - "Command", - "RunningCommand", - "CommandNotFound", - "DEFAULT_ENCODING", - "DoneReadingForever", - "ErrorReturnCode", - "NotYetReadyToRead", - "SignalException", - "ForkException", - "TimeoutException", - "__project_url__", - "__version__", - "__file__", - "args", - "pushd", - "glob", - "contrib", - ]) - - - def __init__(self, globs, baked_args={}): - """ baked_args are defaults for the 'sh' execution context. for - example: - - tmp = sh(_out=StringIO()) - - 'out' would end up in here as an entry in the baked_args dict """ - - self.globs = globs - self.baked_args = baked_args - self.disable_whitelist = False - - def __getitem__(self, k): - # if we first import "_disable_whitelist" from sh, we can import - # anything defined in the global scope of sh.py. this is useful for our - # tests - if k == "_disable_whitelist": - self.disable_whitelist = True - return None - - # we're trying to import something real (maybe), see if it's in our - # global scope - if k in self.whitelist or self.disable_whitelist: - return self.globs[k] - - # somebody tried to be funny and do "from sh import *" - if k == "__all__": - raise RuntimeError("Cannot import * from sh. \ -Please import sh or import programs individually.") - - - # check if we're naming a dynamically generated ReturnCode exception - exc = get_exc_from_name(k) - if exc: - return exc - - - # https://github.com/ipython/ipython/issues/2577 - # https://github.com/amoffat/sh/issues/97#issuecomment-10610629 - if k.startswith("__") and k.endswith("__"): - raise AttributeError - - - # is it a custom builtin? - builtin = getattr(self, "b_" + k, None) - if builtin: - return builtin - - - # is it a command? - cmd = resolve_command(k, self.baked_args) - if cmd: - return cmd - - - # how about an environment variable? - # this check must come after testing if its a command, because on some - # systems, there are an environment variables that can conflict with - # command names. - # https://github.com/amoffat/sh/issues/238 - try: - return os.environ[k] - except KeyError: - pass - - - # nothing found, raise an exception - raise CommandNotFound(k) - - - # methods that begin with "b_" are custom builtins and will override any - # program that exists in our path. this is useful for things like - # common shell builtins that people are used to, but which aren't actually - # full-fledged system binaries - - def b_cd(self, path=None): - if path: - os.chdir(path) - else: - os.chdir(os.path.expanduser('~')) - - def b_which(self, program, paths=None): - return which(program, paths) - - -class Contrib(ModuleType): # pragma: no cover - @classmethod - def __call__(cls, name): - def wrapper1(fn): - - @property - def cmd_getter(self): - cmd = resolve_command(name) - - if not cmd: - raise CommandNotFound(name) - - new_cmd = fn(cmd) - return new_cmd - - setattr(cls, name, cmd_getter) - return fn - - return wrapper1 - - -mod_name = __name__ + ".contrib" -contrib = Contrib(mod_name) -sys.modules[mod_name] = contrib - - -@contrib("git") -def git(orig): # pragma: no cover - """ most git commands play nicer without a TTY """ - cmd = orig.bake(_tty_out=False) - return cmd - -@contrib("sudo") -def sudo(orig): # pragma: no cover - """ a nicer version of sudo that uses getpass to ask for a password, or - allows the first argument to be a string password """ - - prompt = "[sudo] password for %s: " % getpass.getuser() - - def stdin(): - pw = getpass.getpass(prompt=prompt) + "\n" - yield pw - - - def process(args, kwargs): - password = kwargs.pop("password", None) - - if password is None: - pass_getter = stdin() - else: - pass_getter = password.rstrip("\n") + "\n" - - kwargs["_in"] = pass_getter - return args, kwargs - - cmd = orig.bake("-S", _arg_preprocess=process) - return cmd - - - - -def run_repl(env): # pragma: no cover - banner = "\n>> sh v{version}\n>> https://github.com/amoffat/sh\n" - - print(banner.format(version=__version__)) - while True: - try: - line = raw_input("sh> ") - except (ValueError, EOFError): - break - - try: - exec(compile(line, "", "single"), env, env) - except SystemExit: - break - except: - print(traceback.format_exc()) - - # cleans up our last line - print("") - - - - -# this is a thin wrapper around THIS module (we patch sys.modules[__name__]). -# this is in the case that the user does a "from sh import whatever" -# in other words, they only want to import certain programs, not the whole -# system PATH worth of commands. in this case, we just proxy the -# import lookup to our Environment class -class SelfWrapper(ModuleType): - def __init__(self, self_module, baked_args={}): - # this is super ugly to have to copy attributes like this, - # but it seems to be the only way to make reload() behave - # nicely. if i make these attributes dynamic lookups in - # __getattr__, reload sometimes chokes in weird ways... - for attr in ["__builtins__", "__doc__", "__file__", "__name__", "__package__"]: - setattr(self, attr, getattr(self_module, attr, None)) - - # python 3.2 (2.7 and 3.3 work fine) breaks on osx (not ubuntu) - # if we set this to None. and 3.3 needs a value for __path__ - self.__path__ = [] - self.__self_module = self_module - self.__env = Environment(globals(), baked_args=baked_args) - - def __getattr__(self, name): - return self.__env[name] - - def __call__(self, **kwargs): - """ returns a new SelfWrapper object, where all commands spawned from it - have the baked_args kwargs set on them by default """ - baked_args = self.__env.baked_args.copy() - baked_args.update(kwargs) - new_mod = self.__class__(self.__self_module, baked_args) - - # inspect the line in the parent frame that calls and assigns the new sh - # variable, and get the name of the new variable we're assigning to. - # this is very brittle and pretty much a sin. but it works in 99% of - # the time and the tests pass - # - # the reason we need to do this is because we need to remove the old - # cached module from sys.modules. if we don't, it gets re-used, and any - # old baked params get used, which is not what we want - parent = inspect.stack()[1] - code = parent[4][0].strip() - parsed = ast.parse(code) - module_name = parsed.body[0].targets[0].id - - if module_name == __name__: - raise RuntimeError("Cannot use the name 'sh' as an execution context") - - sys.modules.pop(module_name, None) - - return new_mod - - -def in_importlib(frame): - """ helper for checking if a filename is in importlib guts """ - return frame.f_code.co_filename == "" - - -def register_importer(): - """ registers our fancy importer that can let us import from a module name, - like: - - import sh - tmp = sh() - from tmp import ls - """ - - def test(importer): - return importer.__class__.__name__ == ModuleImporterFromVariables.__name__ - already_registered = any([True for i in sys.meta_path if test(i)]) - - if not already_registered: - importer = ModuleImporterFromVariables( - restrict_to=["SelfWrapper"], - ) - sys.meta_path.insert(0, importer) - - return not already_registered - -def fetch_module_from_frame(name, frame): - mod = frame.f_locals.get(name, frame.f_globals.get(name, None)) - return mod - -class ModuleImporterFromVariables(object): - """ a fancy importer that allows us to import from a variable that was - recently set in either the local or global scope, like this: - - sh2 = sh(_timeout=3) - from sh2 import ls - - """ - - def __init__(self, restrict_to=None): - self.restrict_to = set(restrict_to or set()) - - - def find_module(self, mod_fullname, path=None): - """ mod_fullname doubles as the name of the VARIABLE holding our new sh - context. for example: - - derp = sh() - from derp import ls - - here, mod_fullname will be "derp". keep that in mind as we go throug - the rest of this function """ - - parent_frame = inspect.currentframe().f_back - while in_importlib(parent_frame): - parent_frame = parent_frame.f_back - - # this line is saying "hey, does mod_fullname exist as a name we've - # defind previously?" the purpose of this is to ensure that - # mod_fullname is really a thing we've defined. if we haven't defined - # it before, then we "can't" import from it - module = fetch_module_from_frame(mod_fullname, parent_frame) - if not module: - return None - - # make sure it's a class we're allowed to import from - if module.__class__.__name__ not in self.restrict_to: - return None - - return self - - - def load_module(self, mod_fullname): - parent_frame = inspect.currentframe().f_back - - while in_importlib(parent_frame): - parent_frame = parent_frame.f_back - - module = fetch_module_from_frame(mod_fullname, parent_frame) - - # we HAVE to include the module in sys.modules, per the import PEP. - # older verions of python were more lenient about this being set, but - # not in >= python3.3, unfortunately. this requirement necessitates the - # ugly code in SelfWrapper.__call__ - sys.modules[mod_fullname] = module - module.__loader__ = self - - return module - - -def run_tests(env, locale, args, version, **extra_env): # pragma: no cover - py_version = "python" - py_version += str(version) - - py_bin = which(py_version) - return_code = None - - if py_bin: - print("Testing %s, locale %r" % (py_version.capitalize(), - locale)) - - env["LANG"] = locale - - for k,v in extra_env.items(): - env[k] = str(v) - - cmd = [py_bin, "-W", "ignore", os.path.join(THIS_DIR, "test.py")] + args[1:] - launch = lambda: os.spawnve(os.P_WAIT, cmd[0], cmd, env) - return_code = launch() - - return return_code - - - -# we're being run as a stand-alone script -if __name__ == "__main__": # pragma: no cover - def parse_args(): - from optparse import OptionParser - - parser = OptionParser() - parser.add_option("-e", "--envs", dest="envs", action="append") - parser.add_option("-l", "--locales", dest="constrain_locales", action="append") - options, args = parser.parse_args() - - envs = options.envs or [] - constrain_locales = options.constrain_locales or [] - - return args, envs, constrain_locales - - # these are essentially restrictions on what envs/constrain_locales to restrict to for - # the tests. if they're empty lists, it means use all available - args, constrain_versions, constrain_locales = parse_args() - action = None - if args: - action = args[0] - - if action in ("test", "travis"): - import test - coverage = None - if test.HAS_UNICODE_LITERAL: - import coverage - - env = os.environ.copy() - env["SH_TESTS_RUNNING"] = "1" - if coverage: - test.append_module_path(env, coverage) - - # if we're testing locally, run all versions of python on the system - if action == "test": - all_versions = ("2.6", "2.7", "3.1", "3.2", "3.3", "3.4", "3.5") - - # if we're testing on travis, just use the system's default python, - # since travis will spawn a vm per python version in our .travis.yml - # file - elif action == "travis": - v = sys.version_info - sys_ver = "%d.%d" % (v[0], v[1]) - all_versions = (sys_ver,) - - all_locales = ("en_US.UTF-8", "C") - i = 0 - for locale in all_locales: - if constrain_locales and locale not in constrain_locales: - continue - - for version in all_versions: - if constrain_versions and version not in constrain_versions: - continue - - env_copy = env.copy() - exit_code = run_tests(env_copy, locale, args, version, - SH_TEST_RUN_IDX=i) - - if exit_code is None: - print("Couldn't find %s, skipping" % version) - - elif exit_code != 0: - print("Failed for %s, %s" % (version, locale)) - exit(1) - - i += 1 - - ran_versions = ",".join(all_versions) - print("Tested Python versions: %s" % ran_versions) - - else: - env = Environment(globals()) - run_repl(env) - -# we're being imported from somewhere -else: - self = sys.modules[__name__] - sys.modules[__name__] = SelfWrapper(self) - register_importer() - diff --git a/src/program/lwaftr/tests/subcommands/bench_test.py b/src/program/lwaftr/tests/subcommands/bench_test.py index 852837fa2e..8b1056a5a2 100644 --- a/src/program/lwaftr/tests/subcommands/bench_test.py +++ b/src/program/lwaftr/tests/subcommands/bench_test.py @@ -4,7 +4,7 @@ import unittest -from lib.test_env import (BENCHMARK_FILENAME, BENCHMARK_PATH, DATA_DIR, +from test_env import (BENCHMARK_FILENAME, BENCHMARK_PATH, DATA_DIR, BENCHDATA_DIR, SNABB_CMD, BaseTestCase) diff --git a/src/program/lwaftr/tests/subcommands/check_test.py b/src/program/lwaftr/tests/subcommands/check_test.py index d4670b64f5..4fba90defc 100644 --- a/src/program/lwaftr/tests/subcommands/check_test.py +++ b/src/program/lwaftr/tests/subcommands/check_test.py @@ -4,7 +4,7 @@ import unittest -from lib.test_env import COUNTERS_DIR, DATA_DIR, SNABB_CMD, BaseTestCase +from test_env import COUNTERS_DIR, DATA_DIR, SNABB_CMD, BaseTestCase class TestCheck(BaseTestCase): diff --git a/src/program/lwaftr/tests/subcommands/loadtest_test.py b/src/program/lwaftr/tests/subcommands/loadtest_test.py index 4538b16bdb..9dc09b86ab 100644 --- a/src/program/lwaftr/tests/subcommands/loadtest_test.py +++ b/src/program/lwaftr/tests/subcommands/loadtest_test.py @@ -8,8 +8,7 @@ import unittest -from lib.test_env import ( - BENCHDATA_DIR, DATA_DIR, SNABB_CMD, BaseTestCase, nic_names) +from test_env import BENCHDATA_DIR, DATA_DIR, SNABB_CMD, BaseTestCase, nic_names SNABB_PCI0, SNABB_PCI1 = nic_names() diff --git a/src/program/lwaftr/tests/subcommands/monitor_test.py b/src/program/lwaftr/tests/subcommands/monitor_test.py index fe8c605aa2..29c2999546 100644 --- a/src/program/lwaftr/tests/subcommands/monitor_test.py +++ b/src/program/lwaftr/tests/subcommands/monitor_test.py @@ -7,7 +7,7 @@ import unittest -from lib.test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names, tap_name +from test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names, tap_name SNABB_PCI0 = nic_names()[0] diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py index 751ee438e0..a48dd0d3a5 100644 --- a/src/program/lwaftr/tests/subcommands/run_test.py +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -4,7 +4,7 @@ import unittest -from lib.test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names +from test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names SNABB_PCI0, SNABB_PCI1 = nic_names() diff --git a/src/program/lwaftr/tests/lib/test_env.py b/src/program/lwaftr/tests/test_env.py similarity index 100% rename from src/program/lwaftr/tests/lib/test_env.py rename to src/program/lwaftr/tests/test_env.py From 44617fcd4e1e5f93df9c1e5629737f5feac60fd7 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 13 Mar 2017 17:05:52 +0100 Subject: [PATCH 589/631] Convert the "lwaftr config" tests to Python (#768) --- src/program/lwaftr/tests/config/selftest.sh | 12 - .../lwaftr/tests/config/test-config-add.sh | 34 --- .../tests/config/test-config-get-state.sh | 37 --- .../lwaftr/tests/config/test-config-get.sh | 39 --- .../lwaftr/tests/config/test-config-listen.sh | 70 ----- .../lwaftr/tests/config/test-config-remove.sh | 34 --- .../lwaftr/tests/config/test-config-set.sh | 48 ---- src/program/lwaftr/tests/config/test_env.sh | 40 --- .../lwaftr/tests/subcommands/config_test.py | 256 ++++++++++++++++++ src/program/lwaftr/tests/test_env.py | 24 +- 10 files changed, 272 insertions(+), 322 deletions(-) delete mode 100755 src/program/lwaftr/tests/config/selftest.sh delete mode 100755 src/program/lwaftr/tests/config/test-config-add.sh delete mode 100755 src/program/lwaftr/tests/config/test-config-get-state.sh delete mode 100755 src/program/lwaftr/tests/config/test-config-get.sh delete mode 100755 src/program/lwaftr/tests/config/test-config-listen.sh delete mode 100755 src/program/lwaftr/tests/config/test-config-remove.sh delete mode 100755 src/program/lwaftr/tests/config/test-config-set.sh delete mode 100755 src/program/lwaftr/tests/config/test_env.sh create mode 100644 src/program/lwaftr/tests/subcommands/config_test.py diff --git a/src/program/lwaftr/tests/config/selftest.sh b/src/program/lwaftr/tests/config/selftest.sh deleted file mode 100755 index 821e516e6c..0000000000 --- a/src/program/lwaftr/tests/config/selftest.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -# Pass TEST_DIR and CONFIG_TEST_DIR to the invoked scripts. -export TEST_DIR="./program/lwaftr/tests" -export CONFIG_TEST_DIR=${TEST_DIR}/config - -${CONFIG_TEST_DIR}/test-config-get.sh || exit $? -${CONFIG_TEST_DIR}/test-config-set.sh || exit $? -${CONFIG_TEST_DIR}/test-config-add.sh || exit $? -${CONFIG_TEST_DIR}/test-config-remove.sh || exit $? -${CONFIG_TEST_DIR}/test-config-get-state.sh || exit $? -${CONFIG_TEST_DIR}/test-config-listen.sh || exit $? diff --git a/src/program/lwaftr/tests/config/test-config-add.sh b/src/program/lwaftr/tests/config/test-config-add.sh deleted file mode 100755 index 7825c93630..0000000000 --- a/src/program/lwaftr/tests/config/test-config-add.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -## This adds a softwire section and then checks it can be got -## back and that all the values are as they should be. - -TEST_NAME="config add" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -# CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -# Come up with a name for the lwaftr. -SNABB_NAME=lwaftr-$$ - -# Start the bench command. -start_lwaftr_bench $SNABB_NAME - -# IP to test with. -TEST_SOFTWIRE="{ ipv4 1.2.3.4; psid 72; b4-ipv6 ::1; br 1; }" -./snabb config add "$SNABB_NAME" "/softwire-config/binding-table/softwire" "$TEST_SOFTWIRE" - -# Check it can get this just fine. -./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=1.2.3.4][psid=72] &> /dev/null -assert_equal $? 0 - -# Test that the b4-ipv4 is correct. -ADDED_B4_IPV4="`./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=1.2.3.4][psid=72]/b4-ipv6`" -assert_equal "$ADDED_B4_IPV4" "::1" - -stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-get-state.sh b/src/program/lwaftr/tests/config/test-config-get-state.sh deleted file mode 100755 index bbbcff1882..0000000000 --- a/src/program/lwaftr/tests/config/test-config-get-state.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -## This makes various queries to snabb config get-state to verify -## that it will run and produce values. The script has no way of -## validating the accuracy of the values, but it'll check it works. - -TEST_NAME="config get state" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -# CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -# Come up with a name for the lwaftr. -SNABB_NAME=lwaftr-$$ - -# Start the bench command. -start_lwaftr_bench $SNABB_NAME - -# Select a few at random which should have non-zero results. -IN_IPV4="`./snabb config get-state $SNABB_NAME /softwire-state/in-ipv4-bytes`" -if [[ "$IN_IPV4" == "0" ]]; then - exit_on_error "Counter should not show zero." -fi - -OUT_IPV4="`./snabb config get-state $SNABB_NAME /softwire-state/out-ipv4-bytes`" -if [[ "$IN_IPV4" == "0" ]]; then - exit_on_error "Counter should not show zero." -fi - -./snabb config get-state "$SNABB_NAME" / > /dev/null -assert_equal "$?" "0" - -stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-get.sh b/src/program/lwaftr/tests/config/test-config-get.sh deleted file mode 100755 index b91500bd20..0000000000 --- a/src/program/lwaftr/tests/config/test-config-get.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -## Test querying from a known config. The test is dependent on the values -## in the test data files, however this allows for testing basic "getting". -## It performs numerous gets on different paths. - -TEST_NAME="config get" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -# CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -# Come up with a name for the lwaftr. -SNABB_NAME=lwaftr-$$ - -# Start the bench command. -start_lwaftr_bench $SNABB_NAME - -# Check we can get a known value from the config. -INTERNAL_IP="`./snabb config get $SNABB_NAME /softwire-config/internal-interface/ip`" -assert_equal "$INTERNAL_IP" "8:9:a:b:c:d:e:f" - -EXTERNAL_IP="`./snabb config get $SNABB_NAME /softwire-config/external-interface/ip`" -assert_equal "$EXTERNAL_IP" "10.10.10.10" - -BT_B4_IPV6="`./snabb config get $SNABB_NAME /softwire-config/binding-table/softwire[ipv4=178.79.150.233][psid=7850]/b4-ipv6`" -assert_equal "$BT_B4_IPV6" "127:11:12:13:14:15:16:128" - -# Finally test getting a value from the ietf-softwire schema. -IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=127:22:33:44:55:66:77:128]/binding-ipv4-addr" -BINDING_IPV4="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" -assert_equal "$?" "0" -assert_equal "$BINDING_IPV4" "178.79.150.15" - -stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-listen.sh b/src/program/lwaftr/tests/config/test-config-listen.sh deleted file mode 100755 index db0da45554..0000000000 --- a/src/program/lwaftr/tests/config/test-config-listen.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -## This checks it can listen, send a command and get a response. -## It only tests the socket method of communicating with the listen -## command due to the difficulties of testing interactive scripts. - -TEST_NAME="config listen" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -# Verify we have the "nc" tool, used to communicate with sockets, -# and the "python" interpreter available. -# If we don't have them, we just have to skip this test. -check_commands_available "$TEST_NAME" nc python - -# CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -# Come up with a name for the lwaftr. -SNABB_NAME=lwaftr-$$ - -# Start the bench command. -start_lwaftr_bench $SNABB_NAME - -# Start the listen command with a socket. -SOCKET_PATH="/tmp/snabb-test-listen-sock-$SNABB_NAME" -./snabb config listen --socket "$SOCKET_PATH" "$SNABB_NAME" &> /dev/null & - -# Wait a short while for the socket to be created; it shouldn't take long. -sleep 1 - -# Create input and output FIFOs to communicate. -LISTEN_IN=$(mktemp -u) -LISTEN_OUT=$(mktemp -u) -mkfifo "$LISTEN_IN" -mkfifo "$LISTEN_OUT" - -# Start a communication with the listen program. -(cat "$LISTEN_IN" | nc -U "$SOCKET_PATH" > "$LISTEN_OUT") & - -# Get the PID of nc so it can be easily stopped later. -NC_PID=$! - -# Send a get command. -GET_CMD="{ \"id\": \"0\", \"verb\": \"get\", \"path\": \"/routes/route[addr=1.2.3.4]/port\" }" -echo "$GET_CMD" > "$LISTEN_IN" - -# Sleep a short amount of time to let it respond; -# one second should be more than plenty. -sleep 1 - -# Read the response from the listen command. -GET_CMD_RESPONSE=$(cat "$LISTEN_OUT") - -# Check the response as best I can, I'll use python as it's common to most. -PARSED_GET_RESPONSE=$(echo $GET_CMD_RESPONSE | python -c " -import json, sys - -print(json.loads(sys.stdin.read(200))[\"status\"])" -) - -# Test the status is "ok". -assert_equal "$PARSED_GET_RESPONSE" "ok" - -# Finally end all the programs we've spawned. -stop_if_running "$NC_PID" -stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-remove.sh b/src/program/lwaftr/tests/config/test-config-remove.sh deleted file mode 100755 index 32cfc6cbc4..0000000000 --- a/src/program/lwaftr/tests/config/test-config-remove.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -## This adds a softwire section and then checks it can be got -## back and that all the values are as they should be. - -TEST_NAME="config remove" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -# CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -# Come up with a name for the lwaftr. -SNABB_NAME=lwaftr-$$ - -# Start the bench command. -start_lwaftr_bench $SNABB_NAME - -# Verify that the thing we want to remove actually exists. -./snabb config get "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null -assert_equal "$?" "0" - -# Remove it. -./snabb config remove "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null -assert_equal "$?" "0" - -# Verify we can't find it. -./snabb config get "$SNABB_NAME" /softwire-config/binding-table/softwire[ipv4=178.79.150.2][psid=7850]/ &> /dev/null || true -assert_equal "$?" "0" - -stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test-config-set.sh b/src/program/lwaftr/tests/config/test-config-set.sh deleted file mode 100755 index f3813f08fb..0000000000 --- a/src/program/lwaftr/tests/config/test-config-set.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -## This checks you can set values, it'll then perform a get to -## verify the value set is the value that is got too. - -TEST_NAME="config set" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -# CONFIG_TEST_DIR is also set by the caller. -source ${CONFIG_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -# Come up with a name for the lwaftr. -SNABB_NAME=lwaftr-$$ - -# Start the bench command. -start_lwaftr_bench $SNABB_NAME - -# IP to test with. -TEST_IPV4="208.118.235.148" -./snabb config set "$SNABB_NAME" "/softwire-config/external-interface/ip" "$TEST_IPV4" -SET_IP="`./snabb config get \"$SNABB_NAME\" \"/softwire-config/external-interface/ip\"`" -assert_equal "$SET_IP" "$TEST_IPV4" - -# Set a value in a list -TEST_IPV6="::1" -TEST_IPV4="178.79.150.15" -TEST_PSID="0" -./snabb config set "$SNABB_NAME" "/softwire-config/binding-table/softwire[ipv4=$TEST_IPV4][psid=$TEST_PSID]/b4-ipv6" "$TEST_IPV6" -SET_IP="`./snabb config get \"$SNABB_NAME\" \"/softwire-config/binding-table/softwire[ipv4=$TEST_IPV4][psid=$TEST_PSID]/b4-ipv6\"`" -assert_equal "$SET_IP" "$TEST_IPV6" - -# Check that the value we just set is the same in the IETF schema. -# We actually need to look this up backwards, let's just check the same -# IPv4 address as was used to set it above. -IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=::1]/binding-ipv4-addr" -IPV4_ADDR="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" -assert_equal "$IPV4_ADDR" "$TEST_IPV4" - -# Also check the portset, the IPv4 address alone isn't unique. -IETF_PATH="/softwire-config/binding/br/br-instances/br-instance[id=1]/binding-table/binding-entry[binding-ipv6info=::1]/port-set/psid" -PSID="`./snabb config get --schema=ietf-softwire $SNABB_NAME $IETF_PATH`" -assert_equal "$PSID" "$TEST_PSID" - -stop_lwaftr_bench diff --git a/src/program/lwaftr/tests/config/test_env.sh b/src/program/lwaftr/tests/config/test_env.sh deleted file mode 100755 index 4905950848..0000000000 --- a/src/program/lwaftr/tests/config/test_env.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# TEST_DIR is set by the caller. -source ${TEST_DIR}/common.sh || exit $? - -DATA_DIR="${TEST_DIR}/data" -BENCHDATA_DIR="${TEST_DIR}/benchdata" - -# Start the "lwaftr bench" process. The first parameter is the command name. -# The process should end when the script ends; however, if something goes wrong -# and it doesn't end correctly, a duration is set to prevent it running indefinitely. -function start_lwaftr_bench { - ./snabb lwaftr bench --reconfigurable --bench-file /dev/null --name "$1" \ - --duration 30 \ - program/lwaftr/tests/data/icmp_on_fail.conf \ - program/lwaftr/tests/benchdata/ipv{4,6}-0550.pcap &> /dev/null & - - # It takes a little time for lwaftr to properly start. - sleep 2 -} - -function stop_lwaftr_bench { - # Get the job number for lwaftr bench. - local jobid=`jobs | grep -i "lwaftr bench" | awk '{print $1}' | tr -d '[]+'` - kill "%$jobid" - # Wait until it's shutdown. - wait &> /dev/null -} - -function stop_if_running { - # Check if it's running, if not, job done. - kill -0 $1 &> /dev/null - if [[ $? -ne 0 ]]; then - return - fi - - # It's running, try and close it nicely. - kill $1 - wait &> /dev/null -} diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py new file mode 100644 index 0000000000..393ec8e7d9 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -0,0 +1,256 @@ +""" +Test the "snabb lwaftr config" subcommand. Does not need NIC names because +it uses the "bench" subcommand. +""" + +import json +import os +from signal import SIGTERM +import socket +from subprocess import PIPE, Popen +import time +import unittest + +from test_env import BENCHDATA_DIR, DATA_DIR, ENC, SNABB_CMD, BaseTestCase + + +DAEMON_PROC_NAME = 'config_test_daemon' +DAEMON_ARGS = ( + str(SNABB_CMD), 'lwaftr', 'bench', '--reconfigurable', + '--bench-file', '/dev/null', + '--name', DAEMON_PROC_NAME, + str(DATA_DIR / 'icmp_on_fail.conf'), + str(BENCHDATA_DIR / 'ipv4-0550.pcap'), + str(BENCHDATA_DIR / 'ipv6-0550.pcap'), +) +SOCKET_PATH = '/tmp/snabb-lwaftr-listen-sock-%s' % DAEMON_PROC_NAME + + +class TestConfigGet(BaseTestCase): + """ + Test querying from a known config, testing basic "getting". + It performs numerous gets on different paths. + """ + + daemon_args = DAEMON_ARGS + wait_for_daemon_startup = True + + config_args = (str(SNABB_CMD), 'config', 'get', DAEMON_PROC_NAME) + + def test_get_internal_iface(self): + cmd_args = list(self.config_args) + cmd_args.append('/softwire-config/internal-interface/ip') + output = self.run_cmd(cmd_args) + self.assertEqual( + output.strip(), b'8:9:a:b:c:d:e:f', + '\n'.join(('OUTPUT', str(output, ENC)))) + + def test_get_external_iface(self): + cmd_args = list(self.config_args) + cmd_args.append('/softwire-config/external-interface/ip') + output = self.run_cmd(cmd_args) + self.assertEqual( + output.strip(), b'10.10.10.10', + '\n'.join(('OUTPUT', str(output, ENC)))) + + def test_get_b4_ipv6(self): + cmd_args = list(self.config_args) + # Implicit string concatenation, do not add commas. + cmd_args.append( + '/softwire-config/binding-table/softwire' + '[ipv4=178.79.150.233][psid=7850]/b4-ipv6') + output = self.run_cmd(cmd_args) + self.assertEqual( + output.strip(), b'127:11:12:13:14:15:16:128', + '\n'.join(('OUTPUT', str(output, ENC)))) + + def test_get_ietf_path(self): + cmd_args = list(self.config_args)[:-1] + cmd_args.extend(( + '--schema=ietf-softwire', DAEMON_PROC_NAME, + # Implicit string concatenation, do not add commas. + '/softwire-config/binding/br/br-instances/' + 'br-instance[id=1]/binding-table/binding-entry' + '[binding-ipv6info=127:22:33:44:55:66:77:128]/binding-ipv4-addr', + )) + output = self.run_cmd(cmd_args) + self.assertEqual( + output.strip(), b'178.79.150.15', + '\n'.join(('OUTPUT', str(output, ENC)))) + + +class TestConfigListen(BaseTestCase): + """ + Test it can listen, send a command and get a response. Only test the + socket method of communicating with the listen command, due to the + difficulties of testing interactive scripts. + """ + + daemon_args = DAEMON_ARGS + wait_for_daemon_startup = True + + listen_args = (str(SNABB_CMD), 'config', 'listen', + '--socket', SOCKET_PATH, DAEMON_PROC_NAME) + + def test_listen(self): + # Start the listen command with a socket. + listen_daemon = Popen(self.listen_args, stdout=PIPE, stderr=PIPE) + # Wait a short while for the socket to be created. + time.sleep(1) + # Send command to and receive response from the listen command. + # (Implicit string concatenation, no summing needed.) + get_cmd = (b'{ "id": "0", "verb": "get",' + b' "path": "/routes/route[addr=1.2.3.4]/port" }\n') + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.connect(SOCKET_PATH) + sock.sendall(get_cmd) + resp = str(sock.recv(200), ENC) + finally: + sock.close() + status = json.loads(resp)['status'] + self.assertEqual(status, 'ok') + # Terminate the listen command. + listen_daemon.terminate() + ret_code = listen_daemon.wait() + if ret_code not in (0, -SIGTERM): + print('Error terminating daemon:', listen_daemon.args) + print('Exit code:', ret_code) + print('STDOUT\n', str(listen_daemon.stdout.read(), ENC)) + print('STDERR\n', str(listen_daemon.stderr.read(), ENC)) + listen_daemon.stdout.close() + listen_daemon.stderr.close() + os.unlink(SOCKET_PATH) + + +class TestConfigMisc(BaseTestCase): + + daemon_args = DAEMON_ARGS + wait_for_daemon_startup = True + + def get_cmd_args(self, action): + cmd_args = list((str(SNABB_CMD), 'config', 'XXX', DAEMON_PROC_NAME)) + cmd_args[2] = action + return cmd_args + + def test_add(self): + """ + Add a softwire section, get it back and check all the values. + """ + # External IPv4. + add_args = self.get_cmd_args('add') + add_args.extend(( + '/softwire-config/binding-table/softwire', + '{ ipv4 1.2.3.4; psid 72; b4-ipv6 ::1; br 1; }', + )) + self.run_cmd(add_args) + get_args = self.get_cmd_args('get') + get_args.append( + '/softwire-config/binding-table/softwire[ipv4=1.2.3.4][psid=72]') + output = self.run_cmd(get_args) + # run_cmd checks the exit code and fails the test if it is not zero. + get_args[-1] += '/b4-ipv6' + self.assertEqual( + output.strip(), b'b4-ipv6 ::1;', + '\n'.join(('OUTPUT', str(output, ENC)))) + + def test_get_state(self): + get_state_args = self.get_cmd_args('get-state') + # Select a few at random which should have non-zero results. + for query in ( + '/softwire-state/in-ipv4-bytes', + '/softwire-state/out-ipv4-bytes', + ): + cmd_args = list(get_state_args) + cmd_args.append(query) + output = self.run_cmd(cmd_args) + self.assertNotEqual( + output.strip(), b'0', + '\n'.join(('OUTPUT', str(output, ENC)))) + get_state_args.append('/') + self.run_cmd(get_state_args) + # run_cmd checks the exit code and fails the test if it is not zero. + + def test_remove(self): + # Verify that the thing we want to remove actually exists. + get_args = self.get_cmd_args('get') + get_args.append( + # Implicit string concatenation, no summing needed. + '/softwire-config/binding-table/softwire' + '[ipv4=178.79.150.2][psid=7850]/' + ) + self.run_cmd(get_args) + # run_cmd checks the exit code and fails the test if it is not zero. + # Remove it. + remove_args = list(get_args) + remove_args[2] = 'remove' + self.run_cmd(get_args) + # run_cmd checks the exit code and fails the test if it is not zero. + # Verify we cannot find it anymore. + self.run_cmd(get_args) + # run_cmd checks the exit code and fails the test if it is not zero. + + def test_set(self): + """ + Test setting values, then perform a get to verify the value. + """ + # External IPv4. + test_ipv4 = '208.118.235.148' + set_args = self.get_cmd_args('set') + set_args.extend(('/softwire-config/external-interface/ip', test_ipv4)) + self.run_cmd(set_args) + get_args = list(set_args)[:-1] + get_args[2] = 'get' + output = self.run_cmd(get_args) + self.assertEqual( + output.strip(), bytes(test_ipv4, ENC), + '\n'.join(('OUTPUT', str(output, ENC)))) + + # Binding table. + test_ipv4, test_ipv6, test_psid = '178.79.150.15', '::1', '0' + set_args = self.get_cmd_args('set') + # Implicit string concatenation, no summing needed. + set_args.extend(( + '/softwire-config/binding-table/softwire[ipv4=%s][psid=%s]/b4-ipv6' + % (test_ipv4, test_psid), + test_ipv6, + )) + self.run_cmd(set_args) + get_args = list(set_args)[:-1] + get_args[2] = 'get' + output = self.run_cmd(get_args) + self.assertEqual( + output.strip(), bytes(test_ipv6, ENC), + '\n'.join(('OUTPUT', str(output, ENC)))) + + # Check that the value we just set is the same in the IETF schema. + # We actually need to look this up backwards, let's just check the + # same IPv4 address as was used to set it above. + get_args = self.get_cmd_args('get')[:-1] + get_args.extend(( + '--schema=ietf-softwire', DAEMON_PROC_NAME, + # Implicit string concatenation, no summing needed. + '/softwire-config/binding/br/br-instances/' + 'br-instance[id=1]/binding-table/binding-entry' + '[binding-ipv6info=::1]/binding-ipv4-addr', + )) + output = self.run_cmd(get_args) + self.assertEqual( + output.strip(), bytes(test_ipv4, ENC), + '\n'.join(('OUTPUT', str(output, ENC)))) + + # Check the portset: the IPv4 address alone is not unique. + get_args = self.get_cmd_args('get')[:-1] + get_args.extend(( + '--schema=ietf-softwire', DAEMON_PROC_NAME, + # Implicit string concatenation, no summing needed. + '/softwire-config/binding/br/br-instances/br-instance[id=1]/' + 'binding-table/binding-entry[binding-ipv6info=::1]/port-set/psid', + )) + output = self.run_cmd(get_args) + self.assertEqual(output.strip(), bytes(test_psid, ENC), + '\n'.join(('OUTPUT', str(output, ENC)))) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/program/lwaftr/tests/test_env.py b/src/program/lwaftr/tests/test_env.py index 809d7f36f3..2fd7bf99e2 100644 --- a/src/program/lwaftr/tests/test_env.py +++ b/src/program/lwaftr/tests/test_env.py @@ -23,7 +23,9 @@ BENCHMARK_FILENAME = 'benchtest.csv' # Snabb creates the benchmark file in the current directory BENCHMARK_PATH = Path.cwd() / BENCHMARK_FILENAME + COMMAND_TIMEOUT = 10 +ENC = 'utf-8' def nic_names(): @@ -39,7 +41,7 @@ def tap_name(): tap_iface = output.split(b':')[0] if not tap_iface: return None, 'No TAP interface available' - return tap_iface, None + return str(tap_iface, ENC), None class BaseTestCase(unittest.TestCase): @@ -71,14 +73,18 @@ def run_cmd(self, args): except TimeoutExpired: proc.kill() output, errput = proc.communicate() - msg = '\n'.join(('Timeout running command:', str(args), - 'STDOUT', str(output), 'STDERR', str(errput))) + msg = '\n'.join(( + 'Timeout running command:', str(args), + 'STDOUT', str(output, ENC), 'STDERR', str(errput, ENC), + )) self.fail(msg) else: if proc.returncode != 0: - msg = '\n'.join(('Error running command:', str(args), + msg = '\n'.join(( + 'Error running command:', str(args), 'Exit code:', str(proc.returncode), - 'STDOUT', str(output), 'STDERR', str(errput))) + 'STDOUT', str(output, ENC), 'STDERR', str(errput, ENC), + )) self.fail(msg) return output @@ -91,7 +97,9 @@ def tearDownClass(cls): cls.daemon.terminate() ret_code = cls.daemon.wait() if ret_code not in (0, -SIGTERM): - print('Error running daemon:', cls.daemon.args) + print('Error terminating daemon:', cls.daemon.args) print('Exit code:', ret_code) - print('STDOUT\n', cls.daemon.stdout.read()) - print('STDERR\n', cls.daemon.stderr.read()) + print('STDOUT\n', str(cls.daemon.stdout.read(), ENC)) + print('STDERR\n', str(cls.daemon.stderr.read(), ENC)) + cls.daemon.stdout.close() + cls.daemon.stderr.close() From 6a4985b106f6fc27ef654baf1040cbf5d1b954c9 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Thu, 16 Mar 2017 15:28:12 +0100 Subject: [PATCH 590/631] Convert the "lwaftr query" tests to Python (#773) --- src/program/lwaftr/query/selftest.sh | 10 -- src/program/lwaftr/query/tests/test_env.sh | 68 --------- src/program/lwaftr/query/tests/test_query.sh | 42 ------ .../query/tests/test_query_reconfigurable.sh | 50 ------- src/program/lwaftr/tests/common.sh | 74 ---------- .../lwaftr/tests/subcommands/query_test.py | 133 ++++++++++++++++++ src/program/lwaftr/tests/test_env.py | 20 +-- 7 files changed, 140 insertions(+), 257 deletions(-) delete mode 100755 src/program/lwaftr/query/selftest.sh delete mode 100644 src/program/lwaftr/query/tests/test_env.sh delete mode 100755 src/program/lwaftr/query/tests/test_query.sh delete mode 100755 src/program/lwaftr/query/tests/test_query_reconfigurable.sh delete mode 100755 src/program/lwaftr/tests/common.sh create mode 100644 src/program/lwaftr/tests/subcommands/query_test.py diff --git a/src/program/lwaftr/query/selftest.sh b/src/program/lwaftr/query/selftest.sh deleted file mode 100755 index a8b3266824..0000000000 --- a/src/program/lwaftr/query/selftest.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -LWAFTR_DIR="./program/lwaftr" - -# Pass TEST_DIR and QUERY_TEST_DIR to the invoked scripts. -export TEST_DIR="${LWAFTR_DIR}/tests" -export QUERY_TEST_DIR="${LWAFTR_DIR}/query/tests" - -${QUERY_TEST_DIR}/test_query.sh || exit $? -${QUERY_TEST_DIR}/test_query_reconfigurable.sh || exit $? diff --git a/src/program/lwaftr/query/tests/test_env.sh b/src/program/lwaftr/query/tests/test_env.sh deleted file mode 100644 index da083088e7..0000000000 --- a/src/program/lwaftr/query/tests/test_env.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# TEST_DIR is set by the caller. -source ${TEST_DIR}/common.sh || exit $? - -TEST_OUTPUT_FNAME=$(mktemp) - -# Terminate the "lwaftr run" command, remove the output file, and exit. -function query_cleanup { - ps aux | grep $SNABB_PCI0 | grep -v "grep" | awk '{print $2}' | xargs kill 2> /dev/null - ps aux | grep "snabb lwaftr query" | grep -v "grep" | awk '{print $2}' | xargs kill 2> /dev/null - rm -f $TEST_OUTPUT_FNAME - exit -} - -function get_lwaftr_leader { - local pids=$(ps aux | grep "\-\-reconfigurable" | grep $SNABB_PCI0 | grep -v "grep" | awk '{print $2}') - for pid in ${pids[@]}; do - if [[ -d "/var/run/snabb/$pid" ]]; then - echo $pid - fi - done -} - -function get_lwaftr_follower { - local leaders=$(ps aux | grep "\-\-reconfigurable" | grep $SNABB_PCI0 | grep -v "grep" | awk '{print $2}') - for pid in $(ls /var/run/snabb); do - for leader in ${leaders[@]}; do - if [[ -L "/var/run/snabb/$pid/group" ]]; then - local target=$(ls -l /var/run/snabb/$pid/group | awk '{print $11}' | grep -oe "[0-9]\+") - if [[ "$leader" == "$target" ]]; then - echo $pid - fi - fi - done - done -} - -function get_lwaftr_instance { - local pids=$(ps aux | grep $SNABB_PCI0 | awk '{print $2}') - for pid in ${pids[@]}; do - if [[ -d "/var/run/snabb/$pid/apps/lwaftr" ]]; then - echo $pid - fi - done -} - -function test_lwaftr_query { - ./snabb lwaftr query $@ > $TEST_OUTPUT_FNAME - local lineno=`cat $TEST_OUTPUT_FNAME | wc -l` - if [[ $lineno -gt 1 ]]; then - echo "Success: lwaftr query $*" - else - cat $TEST_OUTPUT_FNAME - exit_on_error "Error: lwaftr query $*" - fi -} - -function test_lwaftr_query_no_counters { - ./snabb lwaftr query $@ > $TEST_OUTPUT_FNAME - local lineno=`cat $TEST_OUTPUT_FNAME | wc -l` - if [[ $lineno -eq 1 ]]; then - echo "Success: lwaftr query no counters $*" - else - cat $TEST_OUTPUT_FNAME - exit_on_error "Error: lwaftr query no counters $*" - fi -} diff --git a/src/program/lwaftr/query/tests/test_query.sh b/src/program/lwaftr/query/tests/test_query.sh deleted file mode 100755 index cc816f6bc2..0000000000 --- a/src/program/lwaftr/query/tests/test_query.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -TEST_NAME="lwaftr query" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -check_nics_available "$TEST_NAME" - -# QUERY_TEST_DIR is also set by the caller. -source ${QUERY_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -trap query_cleanup EXIT HUP INT QUIT TERM - -LWAFTR_NAME=lwaftr-$$ -LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf - -# Launch "lwaftr run". -./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF \ - --v4 $SNABB_PCI0 --v6 $SNABB_PCI1 &> lwaftr.log & -sleep 2 - -# Test query all. -test_lwaftr_query -l - -# Test query by pid. -pid=$(get_lwaftr_instance) -if [[ -n "$pid" ]]; then - test_lwaftr_query $pid - test_lwaftr_query $pid "memuse-ipv" - test_lwaftr_query_no_counters $pid counter-never-exists-123 -fi - -# Test query by name. -test_lwaftr_query "--name $LWAFTR_NAME" -test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" -test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" - -exit 0 diff --git a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh b/src/program/lwaftr/query/tests/test_query_reconfigurable.sh deleted file mode 100755 index 288fc9a66d..0000000000 --- a/src/program/lwaftr/query/tests/test_query_reconfigurable.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -TEST_NAME="lwaftr query --reconfigurable" - -# TEST_DIR is set by the caller, and passed onward. -export TEST_DIR -source ${TEST_DIR}/common.sh || exit $? - -check_nics_available "$TEST_NAME" - -# QUERY_TEST_DIR is also set by the caller. -source ${QUERY_TEST_DIR}/test_env.sh || exit $? - -echo "Testing ${TEST_NAME}" - -trap query_cleanup EXIT HUP INT QUIT TERM - -LWAFTR_NAME=lwaftr-$$ -LWAFTR_CONF=${TEST_DIR}/data/no_icmp.conf - -# Launch "lwaftr run". -./snabb lwaftr run --name $LWAFTR_NAME --conf $LWAFTR_CONF --reconfigurable \ - --v4 $SNABB_PCI0 --v6 $SNABB_PCI1 &> lwaftr.log & -sleep 2 - -# Test query all. -test_lwaftr_query -l - -# Test query by leader pid. -pid=$(get_lwaftr_leader) -if [[ -n "$pid" ]]; then - test_lwaftr_query $pid - test_lwaftr_query $pid "memuse-ipv" - test_lwaftr_query_no_counters $pid counter-never-exists-123 -fi - -# Test query by follower pid. -pid=$(get_lwaftr_follower) -if [[ -n "$pid" ]]; then - test_lwaftr_query $pid - test_lwaftr_query $pid "memuse-ipv" - test_lwaftr_query_no_counters $pid counter-never-exists-123 -fi - -# Test query by name. -test_lwaftr_query "--name $LWAFTR_NAME" -test_lwaftr_query "--name $LWAFTR_NAME memuse-ipv" -test_lwaftr_query_no_counters "--name $LWAFTR_NAME counter-never-exists-123" - -exit 0 diff --git a/src/program/lwaftr/tests/common.sh b/src/program/lwaftr/tests/common.sh deleted file mode 100755 index 6366f2517b..0000000000 --- a/src/program/lwaftr/tests/common.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash - -SKIPPED_CODE=43 - -# Show $1 as error message and exit with code $2, or 1 if not passed. -function exit_on_error { - (>&2 echo "$1") - if [[ -n $2 ]]; then - exit $2 - else - exit 1 - fi -} - -# Check that the script is run as root, otherwise exit. -if [[ $EUID != 0 ]]; then - exit_on_error "Tests must be run as root, exiting." -fi - -# If one of the commands from $2 onward is not available, exit -# with code $SKIPPED_CODE mentioning the test name passed in $1. -function check_commands_available { - local test_name=$1 - shift - for cmd in $@; do - which $cmd &> /dev/null - if [[ $? -ne 0 ]]; then - exit_on_error "Cannot find $cmd, skipping $test_name" $SKIPPED_CODE - fi - done -} - -# Check that NIC interfaces are available, otherwise exit with code $SKIPPED_CODE. -function check_nics_available { - if [[ -z "$SNABB_PCI0" ]]; then - exit_on_error "SNABB_PCI0 not set, skipping $1" $SKIPPED_CODE - fi - if [[ -z "$SNABB_PCI1" ]]; then - exit_on_error "SNABB_PCI1 not set, skipping $1" $SKIPPED_CODE - fi -} - -# Check that a file exists, otherwise exit. -# If the second argument is "--remove", remove the file. -function assert_file_exists { - if [[ ! -f "$1" ]]; then - exit_on_error "File $1 does not exists." - fi - if [[ "$2" == "--remove" ]]; then - rm -f "$1" - fi -} - -# Check equality of the first two arguments. -# The third argument will be displayed if the check fails. -# e.g. -# $ assert_equal "yellow "cat" -> error -# $ assert_equal "yellow "cat" "Cat not yellow" -> error with message -# $ assert_equal "banana" "banana" -> nothing (valid) -function assert_equal { - if [[ -z "$2" ]]; then - exit_on_error "assert_equal: not enough arguments." - exit 1 - fi - if [[ "$1" == "$2" ]]; then - return - else - if [[ -z "$3" ]]; then - exit_on_error "Error: $1 != $2" - else - exit_on_error "Error: $3" - fi - fi -} diff --git a/src/program/lwaftr/tests/subcommands/query_test.py b/src/program/lwaftr/tests/subcommands/query_test.py new file mode 100644 index 0000000000..5b7bc45fe3 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/query_test.py @@ -0,0 +1,133 @@ +""" +Test the "snabb lwaftr query" subcommand. Needs NIC names. +""" + +import os +from pathlib import Path +import unittest + +from test_env import DATA_DIR, ENC, SNABB_CMD, BaseTestCase, nic_names + + +DAEMON_PROC_NAME = 'query_test_daemon' +SNABB_PCI0, SNABB_PCI1 = nic_names() +RUN_DIR = Path('/var/run/snabb') + + +@unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') +class TestQueryStandard(BaseTestCase): + + daemon_args = ( + str(SNABB_CMD), 'lwaftr', 'run', + '--name', DAEMON_PROC_NAME, + '--conf', str(DATA_DIR / 'no_icmp.conf'), + '--v4', SNABB_PCI0, + '--v6', SNABB_PCI1, + ) + wait_for_daemon_startup = True + + query_args = (str(SNABB_CMD), 'lwaftr', 'query') + + def test_query_all(self): + all_args = list(self.query_args) + all_args.append('--list-all') + output = self.run_cmd(all_args) + self.assertGreater( + len(output.splitlines()), 1, + '\n'.join(('OUTPUT', str(output, ENC)))) + + def execute_query_test(self, cmd_args): + output = self.run_cmd(cmd_args) + self.assertGreater( + len(output.splitlines()), 1, + '\n'.join(('OUTPUT', str(output, ENC)))) + cmd_args.append('memuse-ipv') + output = self.run_cmd(cmd_args) + self.assertGreater( + len(output.splitlines()), 1, + '\n'.join(('OUTPUT', str(output, ENC)))) + cmd_args[-1] = "no-such-counter" + output = self.run_cmd(cmd_args) + self.assertEqual( + len(output.splitlines()), 1, + '\n'.join(('OUTPUT', str(output, ENC)))) + + def get_lwaftr_pid(self): + output = str(self.run_cmd(('ps', 'aux')), ENC) + pids = [] + for line in output.splitlines(): + if SNABB_PCI0 in line: + pids.append(line.split()[1]) + for pid in pids: + if (RUN_DIR / pid / 'apps' / 'lwaftr').is_dir(): + return pid + + def test_query_by_pid(self): + lwaftr_pid = self.get_lwaftr_pid() + pid_args = list(self.query_args) + pid_args.append(str(lwaftr_pid)) + self.execute_query_test(pid_args) + + def test_query_by_name(self): + name_args = list(self.query_args) + name_args.extend(('--name', DAEMON_PROC_NAME)) + self.execute_query_test(name_args) + + +@unittest.skipUnless(SNABB_PCI0 and SNABB_PCI1, 'NICs not configured') +class TestQueryReconfigurable(TestQueryStandard): + + daemon_args = ( + str(SNABB_CMD), 'lwaftr', 'run', '--reconfigurable', + '--name', DAEMON_PROC_NAME, + '--conf', str(DATA_DIR / 'no_icmp.conf'), + '--v4', SNABB_PCI0, + '--v6', SNABB_PCI1, + ) + wait_for_daemon_startup = True + + def get_all_leader_pids(self): + output = str(self.run_cmd(('ps', 'aux')), ENC) + pids = [] + for line in output.splitlines(): + if ((SNABB_PCI0 in line) and + ('--reconfigurable' in line) and + ('grep' not in line)): + pids.append(line.split()[1]) + return pids + + def get_leader_pid(self): + for pid in self.get_all_leader_pids(): + if (RUN_DIR / pid).is_dir(): + return pid + + def get_follower_pid(self): + leader_pids = self.get_all_leader_pids() + for run_pid in RUN_DIR.iterdir(): + run_pid = run_pid.name + for leader_pid in leader_pids: + group_link = RUN_DIR / run_pid / 'group' + if group_link.is_symlink(): + target = Path(os.readlink(str(group_link))) + # ('/', 'var', 'run', 'snabb', pid, 'group') + target_pid = target.parts[4] + if target_pid == leader_pid: + return run_pid + + def test_query_by_pid(self): + leader_pid = self.get_leader_pid() + if not leader_pid: + self.fail('Could not find the leader PID') + pid_args = list(self.query_args) + pid_args.append(str(leader_pid)) + self.execute_query_test(pid_args) + follower_pid = self.get_follower_pid() + if not follower_pid: + self.fail('Could not find the follower PID') + pid_args = list(self.query_args) + pid_args.append(str(follower_pid)) + self.execute_query_test(pid_args) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/program/lwaftr/tests/test_env.py b/src/program/lwaftr/tests/test_env.py index 2fd7bf99e2..52d502dfdb 100644 --- a/src/program/lwaftr/tests/test_env.py +++ b/src/program/lwaftr/tests/test_env.py @@ -72,20 +72,14 @@ def run_cmd(self, args): output, errput = proc.communicate(timeout=COMMAND_TIMEOUT) except TimeoutExpired: proc.kill() - output, errput = proc.communicate() - msg = '\n'.join(( - 'Timeout running command:', str(args), + proc.communicate() + if proc.returncode != 0: + msg_lines = ( + 'Error running command:', str(args), + 'Exit code:', str(proc.returncode), 'STDOUT', str(output, ENC), 'STDERR', str(errput, ENC), - )) - self.fail(msg) - else: - if proc.returncode != 0: - msg = '\n'.join(( - 'Error running command:', str(args), - 'Exit code:', str(proc.returncode), - 'STDOUT', str(output, ENC), 'STDERR', str(errput, ENC), - )) - self.fail(msg) + ) + self.fail('\n'.join(msg_lines)) return output @classmethod From 4fbaee14161f86bc3c0b671e143beb6712267e3a Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 20 Mar 2017 15:55:28 +0100 Subject: [PATCH 591/631] Reduced the number of globals This patch localizes two constants, adds two imports, removes a stale import, and renames a variable to avoid shadowing an upvalue. --- src/apps/lwaftr/lwaftr.lua | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/apps/lwaftr/lwaftr.lua b/src/apps/lwaftr/lwaftr.lua index d179c19a3a..5bd45e3346 100644 --- a/src/apps/lwaftr/lwaftr.lua +++ b/src/apps/lwaftr/lwaftr.lua @@ -2,7 +2,6 @@ module(..., package.seeall) local bt = require("apps.lwaftr.binding_table") local constants = require("apps.lwaftr.constants") -local dump = require('apps.lwaftr.dump') local icmp = require("apps.lwaftr.icmp") local lwcounter = require("apps.lwaftr.lwcounter") local lwdebug = require("apps.lwaftr.lwdebug") @@ -14,6 +13,8 @@ local ethernet = require("lib.protocol.ethernet") local counter = require("core.counter") local packet = require("core.packet") local lib = require("core.lib") +local link = require("core.link") +local engine = require("core.app") local bit = require("bit") local band, bnot = bit.band, bit.bnot @@ -27,8 +28,8 @@ local is_ipv4_fragment, is_ipv6_fragment = lwutil.is_ipv4_fragment, lwutil.is_ip -- Note whether an IPv4 packet is actually coming from the internet, or from -- a b4 and hairpinned to be re-encapsulated in another IPv6 packet. -PKT_FROM_INET = 1 -PKT_HAIRPINNED = 2 +local PKT_FROM_INET = 1 +local PKT_HAIRPINNED = 2 local debug = lib.getenv("LWAFTR_DEBUG") @@ -490,10 +491,10 @@ local function encapsulate_and_transmit(lwstate, pkt, ipv6_dst, ipv6_src, pkt_sr return transmit(lwstate.o6, pkt) end -local function select_lookup_queue (lwstate, link) - if link == PKT_FROM_INET then +local function select_lookup_queue(lwstate, src_link) + if src_link == PKT_FROM_INET then return lwstate.inet_lookup_queue - elseif link == PKT_HAIRPINNED then + elseif src_link == PKT_HAIRPINNED then return lwstate.hairpin_lookup_queue end assert(false, "Programming error, bad link: " .. link) From 88244123ed96a7105f3423c39ae4ad5ebaf29279 Mon Sep 17 00:00:00 2001 From: Katerina Barone-Adesi Date: Mon, 20 Mar 2017 16:30:32 +0100 Subject: [PATCH 592/631] Require link and engine Add explicit imports to previously implicit globals. This patch also removes a few unused imports. --- src/apps/lwaftr/V4V6.lua | 2 ++ src/apps/lwaftr/ipv4_apps.lua | 2 ++ src/apps/lwaftr/ipv6_apps.lua | 2 ++ src/program/lwaftr/bench/bench.lua | 1 - src/program/lwaftr/check/check.lua | 1 + .../lwaftr/migrate_configuration/migrate_configuration.lua | 2 -- src/program/lwaftr/monitor/monitor.lua | 1 + src/program/lwaftr/query/query.lua | 3 +-- src/program/lwaftr/run/run.lua | 2 +- src/program/lwaftr/run_nohw/run_nohw.lua | 1 + src/program/lwaftr/setup.lua | 1 + src/program/lwaftr/soaktest/soaktest.lua | 1 + 12 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/apps/lwaftr/V4V6.lua b/src/apps/lwaftr/V4V6.lua index 1cce2dc6b3..1bf52b5233 100644 --- a/src/apps/lwaftr/V4V6.lua +++ b/src/apps/lwaftr/V4V6.lua @@ -4,6 +4,8 @@ local constants = require("apps.lwaftr.constants") local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") +local link = require("core.link") +local engine = require("core.app") local transmit, receive = link.transmit, link.receive local rd16, rd32 = lwutil.rd16, lwutil.rd32 diff --git a/src/apps/lwaftr/ipv4_apps.lua b/src/apps/lwaftr/ipv4_apps.lua index 0b9e51e46f..1b8c6c49e2 100644 --- a/src/apps/lwaftr/ipv4_apps.lua +++ b/src/apps/lwaftr/ipv4_apps.lua @@ -14,6 +14,8 @@ local checksum = require("lib.checksum") local packet = require("core.packet") local lib = require("core.lib") local counter = require("core.counter") +local link = require("core.link") +local engine = require("core.app") local receive, transmit = link.receive, link.transmit local wr16, rd32, wr32 = lwutil.wr16, lwutil.rd32, lwutil.wr32 diff --git a/src/apps/lwaftr/ipv6_apps.lua b/src/apps/lwaftr/ipv6_apps.lua index a5538e0471..deddf1f9d5 100644 --- a/src/apps/lwaftr/ipv6_apps.lua +++ b/src/apps/lwaftr/ipv6_apps.lua @@ -14,6 +14,8 @@ local checksum = require("lib.checksum") local packet = require("core.packet") local counter = require("core.counter") local lib = require("core.lib") +local link = require("core.link") +local engine = require("core.app") local bit = require("bit") local ffi = require("ffi") diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 4365eea7a4..ed390d8b79 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -3,7 +3,6 @@ module(..., package.seeall) local app = require("core.app") local config = require("core.config") local lib = require("core.lib") -local numa = require("lib.numa") local csv_stats = require("program.lwaftr.csv_stats") local setup = require("program.lwaftr.setup") local S = require("syscall") diff --git a/src/program/lwaftr/check/check.lua b/src/program/lwaftr/check/check.lua index f936604daf..67852929b5 100644 --- a/src/program/lwaftr/check/check.lua +++ b/src/program/lwaftr/check/check.lua @@ -4,6 +4,7 @@ local lib = require("core.lib") local lwconf = require("apps.lwaftr.conf") local setup = require("program.lwaftr.setup") local util = require("program.lwaftr.check.util") +local engine = require("core.app") local load_requested_counters = util.load_requested_counters local read_counters = util.read_counters diff --git a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua index d2062ecfaf..4ba998102c 100644 --- a/src/program/lwaftr/migrate_configuration/migrate_configuration.lua +++ b/src/program/lwaftr/migrate_configuration/migrate_configuration.lua @@ -2,9 +2,7 @@ module(..., package.seeall) local lib = require('core.lib') local ffi = require("ffi") -local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") -local ipv6 = require("lib.protocol.ipv6") local rangemap = require("apps.lwaftr.rangemap") local ctable = require("lib.ctable") local cltable = require('lib.cltable') diff --git a/src/program/lwaftr/monitor/monitor.lua b/src/program/lwaftr/monitor/monitor.lua index 810c1e7fa5..5da5e020ed 100644 --- a/src/program/lwaftr/monitor/monitor.lua +++ b/src/program/lwaftr/monitor/monitor.lua @@ -7,6 +7,7 @@ local lwtypes = require("apps.lwaftr.lwtypes") local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") local top = require("program.top.top") +local engine = require("core.app") local fatal = lwutil.fatal local select_snabb_instance = top.select_snabb_instance diff --git a/src/program/lwaftr/query/query.lua b/src/program/lwaftr/query/query.lua index 0768ffc3f0..9c3be21724 100644 --- a/src/program/lwaftr/query/query.lua +++ b/src/program/lwaftr/query/query.lua @@ -1,13 +1,12 @@ module(..., package.seeall) -local S = require("syscall") local counter = require("core.counter") local lib = require("core.lib") local lwcounter = require("apps.lwaftr.lwcounter") local lwutil = require("apps.lwaftr.lwutil") local shm = require("core.shm") local top = require("program.top.top") -local app = require("core.app") +local engine = require("core.app") local ps = require("program.ps.ps") local keys, fatal = lwutil.keys, lwutil.fatal diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index bf266a0d9e..6fbec78a82 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -4,10 +4,10 @@ local S = require("syscall") local config = require("core.config") local csv_stats = require("program.lwaftr.csv_stats") local lib = require("core.lib") -local numa = require("lib.numa") local setup = require("program.lwaftr.setup") local ingress_drop_monitor = require("lib.timers.ingress_drop_monitor") local lwutil = require("apps.lwaftr.lwutil") +local engine = require("core.app") local fatal, file_exists = lwutil.fatal, lwutil.file_exists local nic_exists = lwutil.nic_exists diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index a4bf60f52d..4596ea30b3 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -5,6 +5,7 @@ local RawSocket = require("apps.socket.raw").RawSocket local LwAftr = require("apps.lwaftr.lwaftr").LwAftr local lib = require("core.lib") local lwutil = require("apps.lwaftr.lwutil") +local engine = require("core.app") local file_exists = lwutil.file_exists diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 0b3cf916f1..975d2d80cc 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -20,6 +20,7 @@ local ipv4 = require("lib.protocol.ipv4") local ethernet = require("lib.protocol.ethernet") local ipv4_ntop = require("lib.yang.util").ipv4_ntop local S = require("syscall") +local engine = require("core.app") local capabilities = {['ietf-softwire']={feature={'binding', 'br'}}} require('lib.yang.schema').set_default_capabilities(capabilities) diff --git a/src/program/lwaftr/soaktest/soaktest.lua b/src/program/lwaftr/soaktest/soaktest.lua index ad4106288d..ed37085868 100644 --- a/src/program/lwaftr/soaktest/soaktest.lua +++ b/src/program/lwaftr/soaktest/soaktest.lua @@ -4,6 +4,7 @@ local config = require("core.config") local lib = require("core.lib") local lwconf = require('apps.lwaftr.conf') local setup = require("program.lwaftr.setup") +local engine = require("core.app") local long_opts = { duration="D", From 55defa6bca0fa4c6275a49f99608addcd8c6aecb Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Tue, 21 Mar 2017 15:37:43 +0100 Subject: [PATCH 593/631] Create the TAP interface in the "monitor" subcommand test (#776) --- src/program/lwaftr/selftest.sh | 27 --------- .../lwaftr/tests/subcommands/bench_test.py | 2 +- .../lwaftr/tests/subcommands/config_test.py | 5 -- .../lwaftr/tests/subcommands/loadtest_test.py | 1 - .../lwaftr/tests/subcommands/monitor_test.py | 40 ++++++++++--- .../lwaftr/tests/subcommands/query_test.py | 2 - .../lwaftr/tests/subcommands/run_test.py | 4 +- src/program/lwaftr/tests/test_env.py | 56 ++++++++++--------- 8 files changed, 66 insertions(+), 71 deletions(-) delete mode 100755 src/program/lwaftr/selftest.sh diff --git a/src/program/lwaftr/selftest.sh b/src/program/lwaftr/selftest.sh deleted file mode 100755 index 8d51c456b3..0000000000 --- a/src/program/lwaftr/selftest.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e # Exit on any errors - -SKIPPED_CODE=43 -TDIR="program/lwaftr/tests/data/" - -if [[ $EUID != 0 ]]; then - echo "This script must be run as root" - exit 1 -fi - -# The tests require real hardware - -if [ -z "$SNABB_PCI0" ]; then - echo "Skipping tests which require real hardware, SNABB_PCI0 not set" - exit $SKIPPED_CODE -fi - -echo "Testing snabb lwaftr run" -sudo ./snabb lwaftr run -D 0.1 --conf ${TDIR}/icmp_on_fail.conf \ - --on-a-stick "$SNABB_PCI0" - -# This needs to be 1 second, not 0.1 second, or it can mask DMA/setup problems -echo "Testing snabb lwaftr run --reconfigurable" -sudo ./snabb lwaftr run -D 1 --reconfigurable \ - --conf ${TDIR}/icmp_on_fail.conf --on-a-stick "$SNABB_PCI0" diff --git a/src/program/lwaftr/tests/subcommands/bench_test.py b/src/program/lwaftr/tests/subcommands/bench_test.py index 8b1056a5a2..0500a6876d 100644 --- a/src/program/lwaftr/tests/subcommands/bench_test.py +++ b/src/program/lwaftr/tests/subcommands/bench_test.py @@ -25,7 +25,7 @@ def execute_bench_test(self, cmd_args): 'Cannot find {}'.format(BENCHMARK_PATH)) BENCHMARK_PATH.unlink() - def test_bench_standard(self): + def test_bench_not_reconfigurable(self): self.execute_bench_test(self.cmd_args) def test_bench_reconfigurable(self): diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py index 393ec8e7d9..ca94d13937 100644 --- a/src/program/lwaftr/tests/subcommands/config_test.py +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -33,8 +33,6 @@ class TestConfigGet(BaseTestCase): """ daemon_args = DAEMON_ARGS - wait_for_daemon_startup = True - config_args = (str(SNABB_CMD), 'config', 'get', DAEMON_PROC_NAME) def test_get_internal_iface(self): @@ -87,8 +85,6 @@ class TestConfigListen(BaseTestCase): """ daemon_args = DAEMON_ARGS - wait_for_daemon_startup = True - listen_args = (str(SNABB_CMD), 'config', 'listen', '--socket', SOCKET_PATH, DAEMON_PROC_NAME) @@ -126,7 +122,6 @@ def test_listen(self): class TestConfigMisc(BaseTestCase): daemon_args = DAEMON_ARGS - wait_for_daemon_startup = True def get_cmd_args(self, action): cmd_args = list((str(SNABB_CMD), 'config', 'XXX', DAEMON_PROC_NAME)) diff --git a/src/program/lwaftr/tests/subcommands/loadtest_test.py b/src/program/lwaftr/tests/subcommands/loadtest_test.py index 9dc09b86ab..76ef302d53 100644 --- a/src/program/lwaftr/tests/subcommands/loadtest_test.py +++ b/src/program/lwaftr/tests/subcommands/loadtest_test.py @@ -35,7 +35,6 @@ class TestLoadtest(BaseTestCase): str(BENCHDATA_DIR / 'ipv4_and_ipv6_stick_imix.pcap'), 'ALL', 'ALL', SNABB_PCI1, ) - wait_for_daemon_startup = True def test_loadtest(self): output = self.run_cmd(self.loadtest_args) diff --git a/src/program/lwaftr/tests/subcommands/monitor_test.py b/src/program/lwaftr/tests/subcommands/monitor_test.py index 29c2999546..92b911a256 100644 --- a/src/program/lwaftr/tests/subcommands/monitor_test.py +++ b/src/program/lwaftr/tests/subcommands/monitor_test.py @@ -5,28 +5,45 @@ 2. Run "snabb lwaftr monitor" to set the counter and check its output. """ +from random import randint +from subprocess import call, check_call import unittest -from test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names, tap_name +from test_env import DATA_DIR, SNABB_CMD, BaseTestCase, nic_names SNABB_PCI0 = nic_names()[0] -TAP_IFACE, tap_err_msg = tap_name() @unittest.skipUnless(SNABB_PCI0, 'NIC not configured') -@unittest.skipUnless(TAP_IFACE, tap_err_msg) class TestMonitor(BaseTestCase): - daemon_args = ( + daemon_args = [ str(SNABB_CMD), 'lwaftr', 'run', '--bench-file', '/dev/null', '--conf', str(DATA_DIR / 'icmp_on_fail.conf'), '--on-a-stick', SNABB_PCI0, - '--mirror', TAP_IFACE, - ) + '--mirror', # TAP interface name added in setUpClass. + ] monitor_args = (str(SNABB_CMD), 'lwaftr', 'monitor', 'all') - wait_for_daemon_startup = True + + # Use setUpClass to only setup the daemon once for all tests. + @classmethod + def setUpClass(cls): + # Create the TAP interface and append its name to daemon_args + # before calling the superclass' setUpClass, which needs both. + # 'tapXXXXXX' where X is a 0-9 digit. + cls.tap_name = 'tap%s' % randint(100000, 999999) + check_call(('ip', 'tuntap', 'add', cls.tap_name, 'mode', 'tap')) + cls.daemon_args.append(cls.tap_name) + cls.prev_daemon_args = BaseTestCase.daemon_args + BaseTestCase.daemon_args = cls.daemon_args + try: + BaseTestCase.setUpClass() + except Exception: + # Clean up the TAP interface. + call(('ip', 'tuntap', 'delete', cls.tap_name, 'mode', 'tap')) + raise def test_monitor(self): monitor_args = list(self.monitor_args) @@ -37,6 +54,15 @@ def test_monitor(self): self.assertIn(b'255.255.255.255', output, b'\n'.join((b'OUTPUT', output))) + @classmethod + def tearDownClass(cls): + try: + BaseTestCase.tearDownClass() + finally: + BaseTestCase.daemon_args = cls.prev_daemon_args + # Clean up the TAP interface. + call(('ip', 'tuntap', 'delete', cls.tap_name, 'mode', 'tap')) + if __name__ == '__main__': unittest.main() diff --git a/src/program/lwaftr/tests/subcommands/query_test.py b/src/program/lwaftr/tests/subcommands/query_test.py index 5b7bc45fe3..2919fb2f2e 100644 --- a/src/program/lwaftr/tests/subcommands/query_test.py +++ b/src/program/lwaftr/tests/subcommands/query_test.py @@ -24,7 +24,6 @@ class TestQueryStandard(BaseTestCase): '--v4', SNABB_PCI0, '--v6', SNABB_PCI1, ) - wait_for_daemon_startup = True query_args = (str(SNABB_CMD), 'lwaftr', 'query') @@ -84,7 +83,6 @@ class TestQueryReconfigurable(TestQueryStandard): '--v4', SNABB_PCI0, '--v6', SNABB_PCI1, ) - wait_for_daemon_startup = True def get_all_leader_pids(self): output = str(self.run_cmd(('ps', 'aux')), ENC) diff --git a/src/program/lwaftr/tests/subcommands/run_test.py b/src/program/lwaftr/tests/subcommands/run_test.py index a48dd0d3a5..f1a36291dd 100644 --- a/src/program/lwaftr/tests/subcommands/run_test.py +++ b/src/program/lwaftr/tests/subcommands/run_test.py @@ -15,7 +15,7 @@ class TestRun(BaseTestCase): cmd_args = ( str(SNABB_CMD), 'lwaftr', 'run', - '--duration', '0.1', + '--duration', '1', '--bench-file', '/dev/null', '--conf', str(DATA_DIR / 'icmp_on_fail.conf'), '--v4', SNABB_PCI0, @@ -27,7 +27,7 @@ def execute_run_test(self, cmd_args): self.assertIn(b'link report', output, b'\n'.join((b'OUTPUT', output))) - def test_run_standard(self): + def test_run_not_reconfigurable(self): self.execute_run_test(self.cmd_args) def test_run_reconfigurable(self): diff --git a/src/program/lwaftr/tests/test_env.py b/src/program/lwaftr/tests/test_env.py index 52d502dfdb..1c5535940a 100644 --- a/src/program/lwaftr/tests/test_env.py +++ b/src/program/lwaftr/tests/test_env.py @@ -5,7 +5,7 @@ import os from pathlib import Path from signal import SIGTERM -from subprocess import PIPE, Popen, TimeoutExpired, check_output +from subprocess import PIPE, Popen, TimeoutExpired import time import unittest @@ -25,6 +25,7 @@ BENCHMARK_PATH = Path.cwd() / BENCHMARK_FILENAME COMMAND_TIMEOUT = 10 +DAEMON_STARTUP_WAIT = 1 ENC = 'utf-8' @@ -32,30 +33,16 @@ def nic_names(): return os.environ.get('SNABB_PCI0'), os.environ.get('SNABB_PCI1') -def tap_name(): - """ - Return the first TAP interface name if one found: (tap_iface, None). - Return (None, 'No TAP interface available') if none found. - """ - output = check_output(['ip', 'tuntap', 'list']) - tap_iface = output.split(b':')[0] - if not tap_iface: - return None, 'No TAP interface available' - return str(tap_iface, ENC), None - - class BaseTestCase(unittest.TestCase): """ Base class for TestCases. It has a "run_cmd" method and daemon handling, running a subcommand like "snabb lwaftr run" or "bench". - Set the daemon args in "cls.daemon_args" to enable the daemon. - Set "self.wait_for_daemon_startup" to True if it needs time to start up. + Set "daemon_args" to enable the daemon. """ # Override these. daemon_args = () - wait_for_daemon_startup = False # Use setUpClass to only setup the daemon once for all tests. @classmethod @@ -63,8 +50,27 @@ def setUpClass(cls): if not cls.daemon_args: return cls.daemon = Popen(cls.daemon_args, stdout=PIPE, stderr=PIPE) - if cls.wait_for_daemon_startup: - time.sleep(1) + time.sleep(DAEMON_STARTUP_WAIT) + # Check that the daemon started up correctly. + ret_code = cls.daemon.poll() + if ret_code is not None: + cls.reportAndFail('Error starting up daemon:', ret_code) + + @classmethod + def reportAndFail(cls, msg, ret_code): + msg_lines = [ + msg, str(cls.daemon.args), + 'Exit code: %s' % ret_code, + ] + if cls.daemon.stdout.readable: + msg_lines.extend( + ('STDOUT\n', str(cls.daemon.stdout.read(), ENC))) + if cls.daemon.stderr.readable: + msg_lines.extend( + ('STDERR\n', str(cls.daemon.stderr.read(), ENC))) + cls.daemon.stdout.close() + cls.daemon.stderr.close() + cls.fail(cls, '\n'.join(msg_lines)) def run_cmd(self, args): proc = Popen(args, stdout=PIPE, stderr=PIPE) @@ -76,7 +82,7 @@ def run_cmd(self, args): if proc.returncode != 0: msg_lines = ( 'Error running command:', str(args), - 'Exit code:', str(proc.returncode), + 'Exit code: %s' % proc.returncode, 'STDOUT', str(output, ENC), 'STDERR', str(errput, ENC), ) self.fail('\n'.join(msg_lines)) @@ -90,10 +96,8 @@ def tearDownClass(cls): if ret_code is None: cls.daemon.terminate() ret_code = cls.daemon.wait() - if ret_code not in (0, -SIGTERM): - print('Error terminating daemon:', cls.daemon.args) - print('Exit code:', ret_code) - print('STDOUT\n', str(cls.daemon.stdout.read(), ENC)) - print('STDERR\n', str(cls.daemon.stderr.read(), ENC)) - cls.daemon.stdout.close() - cls.daemon.stderr.close() + if ret_code in (0, -SIGTERM): + cls.daemon.stdout.close() + cls.daemon.stderr.close() + else: + cls.reportAndFail('Error terminating daemon:', ret_code) From cf290d1fafb908ac3509f7f192dbee409353a7b4 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Wed, 22 Mar 2017 11:57:55 +0100 Subject: [PATCH 594/631] Improve calling the superclass classmethod in monitor test (#780) --- src/program/lwaftr/tests/subcommands/monitor_test.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/program/lwaftr/tests/subcommands/monitor_test.py b/src/program/lwaftr/tests/subcommands/monitor_test.py index 92b911a256..74fff22554 100644 --- a/src/program/lwaftr/tests/subcommands/monitor_test.py +++ b/src/program/lwaftr/tests/subcommands/monitor_test.py @@ -36,10 +36,8 @@ def setUpClass(cls): cls.tap_name = 'tap%s' % randint(100000, 999999) check_call(('ip', 'tuntap', 'add', cls.tap_name, 'mode', 'tap')) cls.daemon_args.append(cls.tap_name) - cls.prev_daemon_args = BaseTestCase.daemon_args - BaseTestCase.daemon_args = cls.daemon_args try: - BaseTestCase.setUpClass() + super(TestMonitor, cls).setUpClass() except Exception: # Clean up the TAP interface. call(('ip', 'tuntap', 'delete', cls.tap_name, 'mode', 'tap')) @@ -57,9 +55,8 @@ def test_monitor(self): @classmethod def tearDownClass(cls): try: - BaseTestCase.tearDownClass() + super(TestMonitor, cls).tearDownClass() finally: - BaseTestCase.daemon_args = cls.prev_daemon_args # Clean up the TAP interface. call(('ip', 'tuntap', 'delete', cls.tap_name, 'mode', 'tap')) From aa8d105fa84915cdcb7db9df7a889224693d629f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 13:07:27 +0000 Subject: [PATCH 595/631] Fix run_nohw * Add lwAFTR's hairpinning interface. * Move engine.configure before CSVStats setup. * Set correct apps names is CSVStats. --- src/program/lwaftr/run_nohw/run_nohw.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index 4596ea30b3..7a7eb4b281 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -73,11 +73,14 @@ function run(parameters) config.link(c, "b4if.tx -> aftr.v6") config.link(c, "aftr.v4 -> inet.rx") config.link(c, "aftr.v6 -> b4if.rx") + config.link(c, "aftr.hairpin_out -> aftr.hairpin_in") + + engine.configure(c) if verbosity >= 1 then local csv = CSVStatsTimer:new(bench_file) csv:add_app("inet", {"tx", "rx"}, { tx = "IPv4 TX", rx = "IPv4 RX" }) - csv:add_app("tob4", {"tx", "rx"}, { tx = "IPv6 TX", rx = "IPv6 RX" }) + csv:add_app("b4if", {"tx", "rx"}, { tx = "IPv6 TX", rx = "IPv6 RX" }) csv:activate() if verbosity >= 2 then @@ -87,7 +90,6 @@ function run(parameters) end end - engine.configure(c) engine.main { report = { showlinks = true; From 88c3f7f8bef6bd79793d81e160f4372e3c3685af Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 15:50:37 +0100 Subject: [PATCH 596/631] Fix pid in CSVStats --- src/program/lwaftr/csv_stats.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/csv_stats.lua b/src/program/lwaftr/csv_stats.lua index 3852876c32..64ed0f7eeb 100644 --- a/src/program/lwaftr/csv_stats.lua +++ b/src/program/lwaftr/csv_stats.lua @@ -48,7 +48,7 @@ function CSVStatsTimer:new(filename, hydra_mode, pid) local o = { hydra_mode=hydra_mode, link_data={}, file=file, period=1, header = hydra_mode and "benchmark,id,score,unit" or "Time (s)" } o.pid = pid or S.getpid() - o.links_by_app = open_link_counters(pid) + o.links_by_app = open_link_counters(o.pid) return setmetatable(o, {__index = CSVStatsTimer}) end From 22784fa56dd04b14040e4fb68356fb6003bbbc34 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 14:33:00 +0100 Subject: [PATCH 597/631] Clean up run_nohw * Replace semicolons for commas * Remove verbosity level 2 in run_nohw. Involved apps in this graph (RawSocket and lwAFTR) do not overwrite their report() method, so nothing is actually reported. --- src/program/lwaftr/run_nohw/run_nohw.lua | 27 +++++++----------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index 7a7eb4b281..8b4ac8e9ac 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -19,28 +19,28 @@ end local function parse_args(args) local verbosity = 0 local conf_file, b4_if, inet_if - local bench_file = 'bench.csv' + local bench_file local handlers = { v = function () verbosity = verbosity + 1 - end; + end, c = function (arg) check(file_exists(arg), "no such file '%s'", arg) conf_file = arg - end; + end, B = function (arg) b4_if = arg - end; + end, I = function (arg) inet_if = arg - end; + end, b = function (arg) bench_file = arg - end; + end, h = function (arg) print(require("program.lwaftr.run_nohw.README_inc")) main.exit(0) - end; + end, } lib.dogetopt(args, handlers, "b:c:B:I:vh", { help = "h", conf = "c", verbose = "v", @@ -53,7 +53,6 @@ local function parse_args(args) return verbosity, conf_file, b4_if, inet_if, bench_file end - function run(parameters) local verbosity, conf_file, b4_if, inet_if, bench_file = parse_args(parameters) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) @@ -82,17 +81,7 @@ function run(parameters) csv:add_app("inet", {"tx", "rx"}, { tx = "IPv4 TX", rx = "IPv4 RX" }) csv:add_app("b4if", {"tx", "rx"}, { tx = "IPv6 TX", rx = "IPv6 RX" }) csv:activate() - - if verbosity >= 2 then - timer.activate(timer.new("report", function () - app.report_apps() - end, 1e9, "repeating")) - end end - engine.main { - report = { - showlinks = true; - } - } + engine.main({report = { showlinks = true }}) end From c7eb0bd896ee4916e692a8e26f36febafed61804 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 14:48:50 +0100 Subject: [PATCH 598/631] Add option 'duration' in run_nohw * Link statistics are only shown up if the program finishes, thus a duration is needed. * Refactor options in run_nohw. --- src/program/lwaftr/run_nohw/run_nohw.lua | 30 +++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index 8b4ac8e9ac..13901e8283 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -17,12 +17,13 @@ local function check(flag, fmt, ...) end local function parse_args(args) - local verbosity = 0 + local opts = { + verbosity = 0, + } local conf_file, b4_if, inet_if - local bench_file local handlers = { v = function () - verbosity = verbosity + 1 + opts.verbosity = opts.verbosity + 1 end, c = function (arg) check(file_exists(arg), "no such file '%s'", arg) @@ -35,26 +36,29 @@ local function parse_args(args) inet_if = arg end, b = function (arg) - bench_file = arg + opts.bench_file = arg end, h = function (arg) print(require("program.lwaftr.run_nohw.README_inc")) main.exit(0) end, + D = function (arg) + opts.duration = assert(tonumber(arg), "duration must be a number") + end, } - lib.dogetopt(args, handlers, "b:c:B:I:vh", { + lib.dogetopt(args, handlers, "b:c:B:I:vhD:", { help = "h", conf = "c", verbose = "v", ["b4-if"] = "B", ["inet-if"] = "I", - ["bench-file"] = "b", + ["bench-file"] = "b", duration = "D", }) check(conf_file, "no configuration specified (--conf/-c)") check(b4_if, "no B4-side interface specified (--b4-if/-B)") check(inet_if, "no Internet-side interface specified (--inet-if/-I)") - return verbosity, conf_file, b4_if, inet_if, bench_file + return conf_file, b4_if, inet_if, opts end function run(parameters) - local verbosity, conf_file, b4_if, inet_if, bench_file = parse_args(parameters) + local conf_file, b4_if, inet_if, opts = parse_args(parameters) local conf = require('apps.lwaftr.conf').load_lwaftr_config(conf_file) local c = config.new() @@ -76,12 +80,16 @@ function run(parameters) engine.configure(c) - if verbosity >= 1 then - local csv = CSVStatsTimer:new(bench_file) + if opts.verbosity >= 1 then + local csv = CSVStatsTimer:new(opts.bench_file) csv:add_app("inet", {"tx", "rx"}, { tx = "IPv4 TX", rx = "IPv4 RX" }) csv:add_app("b4if", {"tx", "rx"}, { tx = "IPv6 TX", rx = "IPv6 RX" }) csv:activate() end - engine.main({report = { showlinks = true }}) + if opts.duration then + engine.main({duration = opts.duration, report = { showlinks = true }}) + else + engine.main() + end end From f9d77e426a57f469f02f9acdd92fb709bf82b526 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 15:45:48 +0100 Subject: [PATCH 599/631] Rework verbosity option --- src/program/lwaftr/run_nohw/run_nohw.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/program/lwaftr/run_nohw/run_nohw.lua b/src/program/lwaftr/run_nohw/run_nohw.lua index 13901e8283..96d3417ef6 100644 --- a/src/program/lwaftr/run_nohw/run_nohw.lua +++ b/src/program/lwaftr/run_nohw/run_nohw.lua @@ -18,12 +18,12 @@ end local function parse_args(args) local opts = { - verbosity = 0, + verbosity = false, } local conf_file, b4_if, inet_if local handlers = { v = function () - opts.verbosity = opts.verbosity + 1 + opts.verbosity = true end, c = function (arg) check(file_exists(arg), "no such file '%s'", arg) @@ -37,6 +37,7 @@ local function parse_args(args) end, b = function (arg) opts.bench_file = arg + opts.verbosity = true end, h = function (arg) print(require("program.lwaftr.run_nohw.README_inc")) @@ -80,7 +81,7 @@ function run(parameters) engine.configure(c) - if opts.verbosity >= 1 then + if opts.verbosity then local csv = CSVStatsTimer:new(opts.bench_file) csv:add_app("inet", {"tx", "rx"}, { tx = "IPv4 TX", rx = "IPv4 RX" }) csv:add_app("b4if", {"tx", "rx"}, { tx = "IPv6 TX", rx = "IPv6 RX" }) From bc9e1bd7d370eb19bd38925ae0acccef7d14954b Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 15:43:13 +0100 Subject: [PATCH 600/631] Update run_nohw README --- src/program/lwaftr/run_nohw/README | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/program/lwaftr/run_nohw/README b/src/program/lwaftr/run_nohw/README index 9a824eedf0..3c8b24667b 100644 --- a/src/program/lwaftr/run_nohw/README +++ b/src/program/lwaftr/run_nohw/README @@ -4,14 +4,15 @@ Usage: run-nohw --help --conf, -c PATH Path to the lwAFTR configuration file. --b4-if, -B NAME Name of the B4-side network interface. --inet-if, -I NAME Name of the Internet-side network interface. - --verbose, -v Be verbose. Can be used multiple times. + --verbose, -v Print CSV stats to stdout. --bench-file, -b FILENAME The file or path name to which benchmark data is - written. A simple filename or relative pathname - will be based on the current directory. Default - is "bench.csv". + written. A simple filename or relative pathname + will be based on the current directory. --help, -h Show this help message. -When the -v option is used at least once, packets on the network interfaces are -counted and recorded, and the corresponding incoming and outgoing packet rates -are written to a file in CSV format, suitable for passing to a graphing program. +When the -v option is used, packets on the network interfaces are counted and +recorded, and the corresponding incoming and outgoing packet rates are print +out to stdout, suitable for passing to a graphing program. + +If bench-file is set, output is printed to a file instead of stdout. From 942cefd6e765c216c7a7b208388d39aa31db92fe Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 15:05:38 +0100 Subject: [PATCH 601/631] Add run_nohw to the list of subprograms Clean up list and add run_nohw. --- src/program/lwaftr/README | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/program/lwaftr/README b/src/program/lwaftr/README index 011ab75b8e..492096fcd7 100644 --- a/src/program/lwaftr/README +++ b/src/program/lwaftr/README @@ -1,14 +1,14 @@ Usage: snabb lwaftr bench snabb lwaftr check - snabb lwaftr run - snabb lwaftr transient - snabb lwaftr compile-binding-table snabb lwaftr generate-binding-table - snabb lwaftr migrate-configuration - snabb lwaftr control snabb lwaftr loadtest + snabb lwaftr migrate-configuration + snabb lwaftr monitor snabb lwaftr query + snabb lwaftr run + snabb lwaftr run_nohw + snabb lwaftr transient Use --help for per-command usage. Example: From 1c2c7d5134780e58bd69f24ee06050b2f405d946 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 17:55:11 +0100 Subject: [PATCH 602/631] Add unit test for run_nohw --- .../lwaftr/tests/subcommands/run_nohw_test.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/program/lwaftr/tests/subcommands/run_nohw_test.py diff --git a/src/program/lwaftr/tests/subcommands/run_nohw_test.py b/src/program/lwaftr/tests/subcommands/run_nohw_test.py new file mode 100644 index 0000000000..3f83c5b7c0 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/run_nohw_test.py @@ -0,0 +1,77 @@ +""" +Test the "snabb lwaftr run_nohw" subcommand. +""" + +import unittest + +from random import randint +from subprocess import call, check_call +from test_env import DATA_DIR, SNABB_CMD, BaseTestCase + +class TestRun(BaseTestCase): + + program = [ + str(SNABB_CMD), 'lwaftr', 'run_nohw', + ] + cmd_args = { + '--duration': '1', + '--bench-file': '/dev/null', + '--conf': str(DATA_DIR / 'icmp_on_fail.conf'), + '--inet-if': '', + '--b4-if': '', + } + veths = [] + + @classmethod + def setUpClass(cls): + cls.create_veth_pair() + + @classmethod + def create_veth_pair(cls): + veth0 = cls.random_veth_name() + veth1 = cls.random_veth_name() + + # Create veth pair. + check_call(('ip', 'link', 'add', veth0, 'type', 'veth', 'peer', \ + 'name', veth1)) + + # Set interfaces up. + check_call(('ip', 'link', 'set', veth0, 'up')) + check_call(('ip', 'link', 'set', veth1, 'up')) + + # Add interface names to class. + cls.veths.append(veth0) + cls.veths.append(veth1) + + @classmethod + def random_veth_name(cls): + return 'veth%s' % randint(10000, 999999) + + def test_run_nohw(self): + self.execute_run_test(self.cmd_args) + + def execute_run_test(self, cmd_args): + self.cmd_args['--inet-if'] = self.veths[0] + self.cmd_args['--b4-if'] = self.veths[1] + output = self.run_cmd(self.build_cmd()) + self.assertIn(b'link report', output, + b'\n'.join((b'OUTPUT', output))) + + def build_cmd(self): + result = self.program + for item in self.cmd_args.items(): + for each in item: + result.append(each) + return result + + @classmethod + def tearDownClass(cls): + cls.remove_veths() + + @classmethod + def remove_veths(cls): + for i in range(0, len(cls.veths), 2): + check_call(('ip', 'link', 'delete', cls.veths[i])) + +if __name__ == '__main__': + unittest.main() From f3a88c32d0223367a7858d91dc78a993c1b39c3f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 22 Mar 2017 18:03:33 +0100 Subject: [PATCH 603/631] Update documentation --- src/program/lwaftr/run_nohw/README | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/program/lwaftr/run_nohw/README b/src/program/lwaftr/run_nohw/README index 3c8b24667b..6fa5a48341 100644 --- a/src/program/lwaftr/run_nohw/README +++ b/src/program/lwaftr/run_nohw/README @@ -11,6 +11,11 @@ Usage: run-nohw --help will be based on the current directory. --help, -h Show this help message. +It runs the lwAFTR over two OS network interfaces. The simplest way to run +it is to create 2 veth pairs and use them as the inet and b4 interfaces. +Flush traffic to the lwAFTR by pushing packets from the other end of the veth +pairs, using a program such as tcpreplay, for example. + When the -v option is used, packets on the network interfaces are counted and recorded, and the corresponding incoming and outgoing packet rates are print out to stdout, suitable for passing to a graphing program. From 248ec2691eed49911fb12409199bb3a874fa0a9b Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Thu, 23 Mar 2017 12:20:21 +0000 Subject: [PATCH 604/631] Rework unit test --- .../lwaftr/tests/subcommands/run_nohw_test.py | 36 ++++++------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/src/program/lwaftr/tests/subcommands/run_nohw_test.py b/src/program/lwaftr/tests/subcommands/run_nohw_test.py index 3f83c5b7c0..ed6d51589e 100644 --- a/src/program/lwaftr/tests/subcommands/run_nohw_test.py +++ b/src/program/lwaftr/tests/subcommands/run_nohw_test.py @@ -1,5 +1,5 @@ """ -Test the "snabb lwaftr run_nohw" subcommand. +Test the "snabb lwaftr run-nohw" subcommand. """ import unittest @@ -10,10 +10,10 @@ class TestRun(BaseTestCase): - program = [ - str(SNABB_CMD), 'lwaftr', 'run_nohw', + cmd_args = [ + str(SNABB_CMD), 'lwaftr', 'run-nohw', ] - cmd_args = { + cmd_options = { '--duration': '1', '--bench-file': '/dev/null', '--conf': str(DATA_DIR / 'icmp_on_fail.conf'), @@ -24,21 +24,14 @@ class TestRun(BaseTestCase): @classmethod def setUpClass(cls): - cls.create_veth_pair() - - @classmethod - def create_veth_pair(cls): veth0 = cls.random_veth_name() veth1 = cls.random_veth_name() - # Create veth pair. check_call(('ip', 'link', 'add', veth0, 'type', 'veth', 'peer', \ 'name', veth1)) - # Set interfaces up. check_call(('ip', 'link', 'set', veth0, 'up')) check_call(('ip', 'link', 'set', veth1, 'up')) - # Add interface names to class. cls.veths.append(veth0) cls.veths.append(veth1) @@ -48,30 +41,21 @@ def random_veth_name(cls): return 'veth%s' % randint(10000, 999999) def test_run_nohw(self): - self.execute_run_test(self.cmd_args) - - def execute_run_test(self, cmd_args): - self.cmd_args['--inet-if'] = self.veths[0] - self.cmd_args['--b4-if'] = self.veths[1] + self.cmd_options['--inet-if'] = self.veths[0] + self.cmd_options['--b4-if'] = self.veths[1] output = self.run_cmd(self.build_cmd()) self.assertIn(b'link report', output, b'\n'.join((b'OUTPUT', output))) def build_cmd(self): - result = self.program - for item in self.cmd_args.items(): - for each in item: - result.append(each) + result = self.cmd_args + for key, value in self.cmd_options.items(): + result.extend((key, value)) return result @classmethod def tearDownClass(cls): - cls.remove_veths() - - @classmethod - def remove_veths(cls): - for i in range(0, len(cls.veths), 2): - check_call(('ip', 'link', 'delete', cls.veths[i])) + check_call(('ip', 'link', 'delete', cls.veths[0])) if __name__ == '__main__': unittest.main() From fee3322fb5141c2e7f124a3b38d4c1e216c5e19f Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 20 Feb 2017 11:46:10 +0000 Subject: [PATCH 605/631] Use snabbvmx query instead of next_hop --- .../snabbvmx/tests/nexthop/selftest.sh | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/program/snabbvmx/tests/nexthop/selftest.sh b/src/program/snabbvmx/tests/nexthop/selftest.sh index f05756fecd..dd41bd049f 100755 --- a/src/program/snabbvmx/tests/nexthop/selftest.sh +++ b/src/program/snabbvmx/tests/nexthop/selftest.sh @@ -52,22 +52,25 @@ fi # Query nexthop for 10 seconds. TIMEOUT=20 -count=0 +COUNT=0 while true; do - output=`./snabb snabbvmx nexthop | egrep -o "[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+:[[:xdigit:]]+"` - mac_v4=`echo "$output" | head -1` - mac_v6=`echo "$output" | tail -1` + output=$(./snabb snabbvmx query | grep "next_hop_mac") + + mac_v4=$(echo $output | sed "s/.*\([^<]\+\)<\/next_hop_mac_v4>.*/\1/") + mac_v6=$(echo $output | sed "s/.*\([^<]\+\)<\/next_hop_mac_v6>.*/\1/") # FIXME: Should return expected MAC addresses. - # Check VM returned something. - if [[ "$mac_v4" != "00:00:00:00:00:00" && - "$mac_v6" != "00:00:00:00:00:00" ]]; then - echo "Resolved MAC inet side: $mac_v4 [OK]" - echo "Resolved MAC b4 side: $mac_v6 [OK]" - exit 0 + # Check VM returned something and it's different than 00:00:00:00:00:00. + if [[ -n "$mac_v4" && -n "$mac_v6" ]]; then + if [[ "$mac_v4" != "00:00:00:00:00:00" && + "$mac_v6" != "00:00:00:00:00:00" ]]; then + echo "Resolved MAC inet side: $mac_v4 [OK]" + echo "Resolved MAC b4 side: $mac_v6 [OK]" + exit 0 + fi fi - if [[ $count == $TIMEOUT ]]; then + if [[ $COUNT == $TIMEOUT ]]; then echo "Could not resolve nexthop" echo "MAC inet side: $mac_v4 [FAILED]" echo "MAC b4 side: $mac_v6 [FAILED]" @@ -75,6 +78,6 @@ while true; do fi # Try again until TIMEOUT. - count=$((count + 1)) + COUNT=$((COUNT + 1)) sleep 1 done From 6edbd9df1cbd2857fb95032bbeb4cac7fe846c83 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 24 Mar 2017 11:13:47 +0000 Subject: [PATCH 606/631] Remove snabbvmx nexthop --- src/program/snabbvmx/nexthop/README | 8 --- src/program/snabbvmx/nexthop/README.inc | 1 - src/program/snabbvmx/nexthop/nexthop.lua | 63 ------------------------ 3 files changed, 72 deletions(-) delete mode 100644 src/program/snabbvmx/nexthop/README delete mode 120000 src/program/snabbvmx/nexthop/README.inc delete mode 100644 src/program/snabbvmx/nexthop/nexthop.lua diff --git a/src/program/snabbvmx/nexthop/README b/src/program/snabbvmx/nexthop/README deleted file mode 100644 index 6193451197..0000000000 --- a/src/program/snabbvmx/nexthop/README +++ /dev/null @@ -1,8 +0,0 @@ -Usage: - nexthop [OPTIONS] - - -h, --help - Print usage information. - -Prints out IPv4 and IPv6 nexthop counter values of an lwAFTR process. These -counters are defined by 'nh_fwd' app. diff --git a/src/program/snabbvmx/nexthop/README.inc b/src/program/snabbvmx/nexthop/README.inc deleted file mode 120000 index 100b93820a..0000000000 --- a/src/program/snabbvmx/nexthop/README.inc +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/program/snabbvmx/nexthop/nexthop.lua b/src/program/snabbvmx/nexthop/nexthop.lua deleted file mode 100644 index bc1b20565f..0000000000 --- a/src/program/snabbvmx/nexthop/nexthop.lua +++ /dev/null @@ -1,63 +0,0 @@ -module(..., package.seeall) - -local S = require("syscall") -local ethernet = require("lib.protocol.ethernet") -local ffi = require("ffi") -local lib = require("core.lib") -local lwutil = require("apps.lwaftr.lwutil") -local shm = require("core.shm") - -local file_exists = lwutil.file_exists - -local macaddress_t = ffi.typeof[[ -struct { uint8_t ether[6]; } -]] - -local long_opts = { - help = "h" -} - -local function usage (code) - print(require("program.snabbvmx.nexthop.README_inc")) - main.exit(code) -end - -local function parse_args (args) - local handlers = {} - function handlers.h (arg) usage(0) end - return lib.dogetopt(args, handlers, "h", long_opts) -end - -local function is_current_process (pid) - return pid == S.getpid() -end - -function run (args) - parse_args(args) - for _, pid in ipairs(shm.children("/")) do - pid = tonumber(pid) - if is_current_process(pid) then - goto continue - end - - -- Print IPv4 next_hop_mac if defined. - local next_hop_mac_v4 = "/"..pid.."/next_hop_mac_v4" - if file_exists(shm.root..next_hop_mac_v4) then - local nh_v4 = shm.open(next_hop_mac_v4, macaddress_t, "readonly") - print(("PID '%d': next_hop_mac for IPv4 is %s"):format( - pid, ethernet:ntop(nh_v4.ether))) - shm.unmap(nh_v4) - end - - -- Print IPv6 next_hop_mac if defined. - local next_hop_mac_v6 = "/"..pid.."/next_hop_mac_v6" - if file_exists(shm.root..next_hop_mac_v6) then - local nh_v6 = shm.open(next_hop_mac_v6, macaddress_t, "readonly") - print(("PID '%d': next_hop_mac for IPv6 is %s"):format( - pid, ethernet:ntop(nh_v6.ether))) - shm.unmap(nh_v6) - end - - ::continue:: - end -end From 20ea537b0b639524a09885158781a44983c709c2 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 24 Mar 2017 12:35:38 +0100 Subject: [PATCH 607/631] Fix softwire validation for config and initial start This fixes a bug where it would error when an invalid softwire was provided on a snabb config add but it was still added. It also adds checking for starting the lwaftr with an invalid config. --- src/apps/config/leader.lua | 2 ++ src/apps/config/support/snabb-softwire-v1.lua | 30 +++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 807cdbc403..6643e561ff 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -69,6 +69,7 @@ function Leader:new (conf) end function Leader:set_initial_configuration (configuration) + self.support.validate_config(configuration) self.current_configuration = configuration self.current_app_graph = self.setup_fn(configuration) self.current_in_place_dependencies = {} @@ -395,6 +396,7 @@ function compute_remove_config_fn (schema_name, path) end function Leader:notify_pre_update (config, verb, path, ...) + self.support.validate_update(config, verb, path, ...) for _,translator in pairs(self.support.translators) do translator.pre_update(config, verb, path, ...) end diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index ff7b750a66..97a2ebd62d 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -16,25 +16,41 @@ local path_mod = require('lib.yang.path') local util = require("lib.yang.util") local generic = require('apps.config.support').generic_schema_config_support --- Validates that the softwire is in the PSID mapping, returns true if the --- softwire is valid (i.e. has a corresponding PSID mapping), false if invalid. -local function validate_softwire_psid_mapping(graph, softwire) +-- Validates that the softwire is in the PSID mapping, if the PSID mapping is +-- missing it will raise an error with an appropriate message. +local function validate_softwire(config, softwire) local function convert_ipv4_to_key(val) local key = corelib.htonl(val) -- Convert endianness return ipv4:pton(ipv4_ntop(key)) end - local bt = graph.apps.lwaftr.arg.softwire_config.binding_table + local bt = config.softwire_config.binding_table local psidmap_key = convert_ipv4_to_key(softwire.key.ipv4) local psidmap_entry = cltable.get(bt.psid_map, psidmap_key) local ip = ipv4_ntop(softwire.key.ipv4) assert(psidmap_entry, "No PSID map for softwire '"..ip.."'") end +local function validate_config(config) + assert(config) + local bt = config.softwire_config.binding_table + for softwire in bt.softwire:iterate() do + validate_softwire(config, softwire) + end +end + +local function validate_update(config, verb, path, data) + -- Validate softwire has PSID Map entry. + if path == "/softwire-config/binding-table/softwire" then + for softwire in data:iterate() do + validate_softwire(config, softwire) + end + end +end + local function add_softwire_entry_actions(app_graph, entries) assert(app_graph.apps['lwaftr']) local ret = {} for entry in entries:iterate() do - validate_softwire_psid_mapping(app_graph, entry) local blob = entries.entry_type() ffi.copy(blob, entry, ffi.sizeof(blob)) local args = {'lwaftr', 'add_softwire_entry', blob} @@ -344,7 +360,7 @@ local function ietf_softwire_translator () end local bt = native_binding_table_from_ietf(arg) return {{'set', {schema='snabb-softwire-v1', - path='/softwire-config/binding-table', + path='/softwire-config/binding-table', config=serialize_binding_table(bt)}}} else -- An update to an existing entry. First, get the existing entry. @@ -589,6 +605,8 @@ end function get_config_support() return { + validate_config = validate_config, + validate_update = validate_update, compute_config_actions = compute_config_actions, update_mutable_objects_embedded_in_app_initargs = update_mutable_objects_embedded_in_app_initargs, From 4fdd8635094b846d571bd08846e7cafe6b51dede Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 24 Mar 2017 13:20:10 +0100 Subject: [PATCH 608/631] Add validation functions to generic leader support --- src/apps/config/support.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/apps/config/support.lua b/src/apps/config/support.lua index d2ecce52f1..f7abe3a613 100644 --- a/src/apps/config/support.lua +++ b/src/apps/config/support.lua @@ -177,6 +177,8 @@ end generic_schema_config_support = { + validate_config = function() end, + validate_update = function () end, compute_config_actions = function( old_graph, new_graph, to_restart, verb, path, ...) return add_restarts(app.compute_config_actions(old_graph, new_graph), From 85659dd3035b86589764a57bfc96f5ebc4d2b185 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 24 Mar 2017 15:12:44 +0100 Subject: [PATCH 609/631] Fix config validation for non --reconfigurable too --- src/apps/config/leader.lua | 1 - src/apps/config/support/snabb-softwire-v1.lua | 2 +- src/program/lwaftr/setup.lua | 8 ++++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 6643e561ff..9c3155b37f 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -69,7 +69,6 @@ function Leader:new (conf) end function Leader:set_initial_configuration (configuration) - self.support.validate_config(configuration) self.current_configuration = configuration self.current_app_graph = self.setup_fn(configuration) self.current_in_place_dependencies = {} diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 97a2ebd62d..80463954ef 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -30,7 +30,7 @@ local function validate_softwire(config, softwire) assert(psidmap_entry, "No PSID map for softwire '"..ip.."'") end -local function validate_config(config) +function validate_config(config) assert(config) local bt = config.softwire_config.binding_table for softwire in bt.softwire:iterate() do diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index 0b3cf916f1..f4aded241e 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -24,6 +24,11 @@ local S = require("syscall") local capabilities = {['ietf-softwire']={feature={'binding', 'br'}}} require('lib.yang.schema').set_default_capabilities(capabilities) +local function validate_config(conf) + local support = require("apps.config.support.snabb_softwire_v1") + return support.validate_config(conf) +end + local function convert_ipv4(addr) if addr ~= nil then return ipv4:pton(ipv4_ntop(addr)) end end @@ -58,6 +63,9 @@ function lwaftr_app(c, conf) error("One or both of the 'next_hop' values must be specified") end + -- Validate configuration file we've been given. + validate_config(conf) + config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = external_interface.reassembly.max_packets, From 6d5f80fb674d8222c177e792179c25d537285a0e Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 24 Mar 2017 16:58:07 +0100 Subject: [PATCH 610/631] Append lwAFTR version number and date to help info --- src/program/lwaftr/lwaftr.lua | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/program/lwaftr/lwaftr.lua b/src/program/lwaftr/lwaftr.lua index a9537ab2f0..d452855f9f 100644 --- a/src/program/lwaftr/lwaftr.lua +++ b/src/program/lwaftr/lwaftr.lua @@ -2,8 +2,24 @@ module(..., package.seeall) local lib = require("core.lib") +-- Retrieves latest version from CHANGELOG.md. +-- Format: ## [Version] - 2017-31-12. +local function latest_version() + local filename = "program/lwaftr/doc/CHANGELOG.md" + local regex = "^## %[([^%]]+)%] %- (%d%d%d%d%-%d%d%-%d%d)" + for line in io.lines(filename) do + local version, date = line:match(regex) + if version and date then return version, date end + end +end + local function show_usage(exit_code) - print(require("program.lwaftr.README_inc")) + local content = require("program.lwaftr.README_inc") + local version, date = latest_version() + if version and date then + content = ("Version: %s (%s)\n\n"):format(version, date)..content + end + print(content) main.exit(exit_code) end From d8953958bec19b35fbb52d610df7283241795af3 Mon Sep 17 00:00:00 2001 From: Nicola Larosa Date: Mon, 27 Mar 2017 10:34:05 +0200 Subject: [PATCH 611/631] Remove the default bench.csv filename for benchmark data (#782) --- src/program/lwaftr/bench/README | 12 +-- src/program/lwaftr/bench/bench.lua | 2 +- src/program/lwaftr/loadtest/README | 18 ++-- src/program/lwaftr/loadtest/loadtest.lua | 85 ++++++++++--------- src/program/lwaftr/run/README | 10 +-- src/program/lwaftr/run/run.lua | 2 +- src/program/lwaftr/run_nohw/README | 9 +- .../lwaftr/tests/subcommands/run_nohw_test.py | 11 ++- src/program/lwaftr/transient/README | 16 ++-- src/program/lwaftr/transient/transient.lua | 3 +- 10 files changed, 87 insertions(+), 81 deletions(-) diff --git a/src/program/lwaftr/bench/README b/src/program/lwaftr/bench/README index 90501ffe2c..61ef88d282 100644 --- a/src/program/lwaftr/bench/README +++ b/src/program/lwaftr/bench/README @@ -12,10 +12,9 @@ Usage: bench CONF IPV4-IN.PCAP IPV6-IN.PCAP Time (s),Decap. MPPS,Decap. Gbps,Encap. MPPS,Encap. Gbps -b FILENAME, --bench-file FILENAME - The file or path name to which benchmark data is - written. A simple filename or relative pathname - will be based on the current directory. Default - is "bench.csv". + The file or path name where benchmark data is to + be written. A simple filename or relative pathname + will be based on the current directory. -D DURATION, --duration DURATION Duration in seconds. -n NAME, --name NAME Sets the name for this program, which will be used @@ -33,5 +32,6 @@ machine that may not have Intel 82599 NICs. Exit when finished. This program is used in the lwAFTR test suite. Packets are counted and recorded, and the corresponding incoming and outgoing -packet rates are written to a file in CSV format, suitable for passing to a -graphing program. +packet rates are written to the stdout in CSV format, suitable for passing +to a graphing program. If bench-file is set, output is written to a file +instead of stdout. diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index ed390d8b79..cb7280195b 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -14,7 +14,7 @@ end function parse_args(args) local handlers = {} - local opts = { bench_file = 'bench.csv' } + local opts = {} local scheduling = {} function handlers.D(arg) opts.duration = assert(tonumber(arg), "duration must be a number") diff --git a/src/program/lwaftr/loadtest/README b/src/program/lwaftr/loadtest/README index 8c705697eb..fdc7d1fb01 100644 --- a/src/program/lwaftr/loadtest/README +++ b/src/program/lwaftr/loadtest/README @@ -9,10 +9,9 @@ Usage: loadtest [OPTIONS] [ [ ].. -p period, --period PERIOD Measure each PERIOD seconds. --bench-file FILENAME - The file or path name to which benchmark data is - written. A simple filename or relative pathname - will be based on the current directory. Default - is "bench.csv". + The file or path name where benchmark data is to + be written. A simple filename or relative pathname + will be based on the current directory. -h, --help Print usage information. @@ -25,10 +24,11 @@ peak bitrate is reached, back down the same way until 0 is reached, and end the test. Packets received on the network interfaces are counted and recorded, and the -corresponding incoming and outgoing packet rates are written to a file in CSV -format, suitable for passing to a graphing program. The NAME values are used -to label the columns. Measurements will be taken each PERIOD seconds, which -defaults to 1. +corresponding incoming and outgoing packet rates are written to standard +output in CSV format, suitable for passing to a graphing program. If +bench-file is set, output is written to a file instead of stdout. The NAME +values are used to label the columns. Measurements will be taken each PERIOD +seconds, which defaults to 1. Examples: transient cap1.pcap tx 01:00.0 diff --git a/src/program/lwaftr/transient/transient.lua b/src/program/lwaftr/transient/transient.lua index 727d27c83e..2dec453c87 100644 --- a/src/program/lwaftr/transient/transient.lua +++ b/src/program/lwaftr/transient/transient.lua @@ -44,8 +44,7 @@ end function parse_args(args) local handlers = {} - local opts = { - bitrate = 10e9, duration = 5, period = 1, bench_file = 'bench.csv' } + local opts = { bitrate = 10e9, duration = 5, period = 1 } function handlers.b(arg) opts.bitrate = assert(tonumber(arg), 'bitrate must be a number') end From 50799485307f3af9db9a442badf3003bdb778323 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Fri, 24 Mar 2017 20:56:59 +0100 Subject: [PATCH 612/631] Reintroduce NDP secondary address test --- src/program/lwaftr/tests/data/add-vlan.sh | 3 ++- .../lwaftr/tests/data/counters/ndp-secondary.lua | 5 +++++ .../tests/data/ndp_incoming_ns_secondary.pcap | Bin 0 -> 126 bytes .../data/vlan/ndp_incoming_ns_secondary.pcap | Bin 0 -> 130 bytes src/program/lwaftr/tests/end-to-end/test_env.sh | 4 ++++ 5 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 src/program/lwaftr/tests/data/counters/ndp-secondary.lua create mode 100644 src/program/lwaftr/tests/data/ndp_incoming_ns_secondary.pcap create mode 100644 src/program/lwaftr/tests/data/vlan/ndp_incoming_ns_secondary.pcap diff --git a/src/program/lwaftr/tests/data/add-vlan.sh b/src/program/lwaftr/tests/data/add-vlan.sh index 0e9fca2f43..3ce40ed452 100755 --- a/src/program/lwaftr/tests/data/add-vlan.sh +++ b/src/program/lwaftr/tests/data/add-vlan.sh @@ -68,9 +68,10 @@ V6=( ipv6-tunneled-incoming-icmpv4-34toobig.pcap ipv6-tunneled-incoming-icmpv4-echo-reply.pcap ipv6-tunneled-incoming-icmpv4-echo-request.pcap + ndp_incoming_ns_secondary.pcap ndp_getna_compound.pcap - ndp_incoming_ns_nonlwaftr.pcap ndp_incoming_ns.pcap + ndp_incoming_ns_nonlwaftr.pcap ndp_ns_and_recap.pcap ndp_outgoing_ns.pcap ndp_outgoing_solicited_na.pcap diff --git a/src/program/lwaftr/tests/data/counters/ndp-secondary.lua b/src/program/lwaftr/tests/data/counters/ndp-secondary.lua new file mode 100644 index 0000000000..c140f2dd1e --- /dev/null +++ b/src/program/lwaftr/tests/data/counters/ndp-secondary.lua @@ -0,0 +1,5 @@ +return { + ["in-ipv6-frag-reassembly-unneeded"] = 1, + ["memuse-ipv4-frag-reassembly-buffer"] = 463571780, + ["memuse-ipv6-frag-reassembly-buffer"] = 464727376, +} diff --git a/src/program/lwaftr/tests/data/ndp_incoming_ns_secondary.pcap b/src/program/lwaftr/tests/data/ndp_incoming_ns_secondary.pcap new file mode 100644 index 0000000000000000000000000000000000000000..c1cd703c1226c53752f8a3385ac9e9f4b9d17004 GIT binary patch literal 126 zcmca|c+)~A1{MYw_+QV!zzF2X_-zc6+r-8Y24sWqe=uNRWMXDvZM&NQ6k$-X`p;+p yRLsP{3^ah1fo=D01`Y;J1}+9}1|9}p20jM*xc3|tJ{3_J|H415gy?F^f Date: Mon, 27 Mar 2017 19:06:28 +0200 Subject: [PATCH 613/631] Lookup psid entries using rangemap for range support This switches to maintaining an instance of the lwaftr's binding table in the leader to allow for looking up PSID entries, specifically those which have a range (i.e. specify an "end-addr"). It still needs to maintain the binding table where possible to prevent it being re-build on every validation (expensive). --- src/apps/config/support/snabb-softwire-v1.lua | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 80463954ef..9c0ddf59e4 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -7,27 +7,39 @@ local equal = require('core.lib').equal local dirname = require('core.lib').dirname local data = require('lib.yang.data') local ipv4_ntop = require('lib.yang.util').ipv4_ntop -local ipv4 = require('lib.protocol.ipv4') local ipv6 = require('lib.protocol.ipv6') local yang = require('lib.yang.yang') local ctable = require('lib.ctable') local cltable = require('lib.cltable') local path_mod = require('lib.yang.path') -local util = require("lib.yang.util") local generic = require('apps.config.support').generic_schema_config_support +local binding_table = require("apps.lwaftr.binding_table") + + +local binding_table_instance +local function get_binding_table(conf) + if binding_table_instance ~= nil then + -- TODO: fix me! + binding_table_instance = nil + return get_binding_table(conf) + else + binding_table_instance = binding_table.load(conf) + end + return binding_table_instance +end -- Validates that the softwire is in the PSID mapping, if the PSID mapping is -- missing it will raise an error with an appropriate message. local function validate_softwire(config, softwire) - local function convert_ipv4_to_key(val) - local key = corelib.htonl(val) -- Convert endianness - return ipv4:pton(ipv4_ntop(key)) - end - local bt = config.softwire_config.binding_table - local psidmap_key = convert_ipv4_to_key(softwire.key.ipv4) - local psidmap_entry = cltable.get(bt.psid_map, psidmap_key) - local ip = ipv4_ntop(softwire.key.ipv4) - assert(psidmap_entry, "No PSID map for softwire '"..ip.."'") + local key = softwire.key.ipv4 + local conf_bt = config.softwire_config.binding_table + local bt = get_binding_table(conf_bt) + local entry = bt.psid_map:lookup(key).value + local ip = ipv4_ntop(key) + + -- Figure out if the PSID map entry is valid. + local reserved_port_bit_count = 16 - entry.shift - entry.psid_length + assert(reserved_port_bit_count ~= 16, "No PSID map for softwire '"..ip.."'") end function validate_config(config) From 6993af8a187f0d30f7026eadb3984078434b1d93 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Mon, 20 Feb 2017 11:46:29 +0000 Subject: [PATCH 614/631] Print out when VM setup is ready --- src/program/snabbvmx/tests/test_env/test_env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/program/snabbvmx/tests/test_env/test_env.sh b/src/program/snabbvmx/tests/test_env/test_env.sh index 5dac08fec3..a57960fecf 100755 --- a/src/program/snabbvmx/tests/test_env/test_env.sh +++ b/src/program/snabbvmx/tests/test_env/test_env.sh @@ -69,6 +69,7 @@ function start_test_env { wait_vm_up $SNABB_TELNET0 # Configure eth0 interface in the VM. + echo -n "Setting up VM..." # Bring up interface. run_telnet $SNABB_TELNET0 "ifconfig eth0 up" >/dev/null @@ -88,4 +89,6 @@ function start_test_env { # Activate IPv4 and IPv6 forwarding. run_telnet $SNABB_TELNET0 "sysctl -w net.ipv4.conf.all.forwarding=1" >/dev/null run_telnet $SNABB_TELNET0 "sysctl -w net.ipv6.conf.all.forwarding=1" >/dev/null + + echo " [OK]" } From 7b557816c68e7c4cacc0c5edbb808af1f54e296a Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 14 Feb 2017 16:19:17 +0000 Subject: [PATCH 615/631] Pass SnabbVMX PID to snabb monitor all --- src/program/snabbvmx/tests/selftest.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/program/snabbvmx/tests/selftest.sh b/src/program/snabbvmx/tests/selftest.sh index f580cf4a78..b4fe5425b1 100755 --- a/src/program/snabbvmx/tests/selftest.sh +++ b/src/program/snabbvmx/tests/selftest.sh @@ -34,8 +34,9 @@ SNABBVMX_ID=xe1 SNABB_TELNET0=5000 VHU_SOCK0=/tmp/vh1a.sock -function monitor { action=$1 - ./snabb lwaftr monitor $action &> /dev/null +function monitor { + local action=$1 pid=$2 + ./snabb lwaftr monitor $action $pid &> monitor.log } function tcpreplay { @@ -269,9 +270,19 @@ function test_ndp_request_to_lwaftr { } function cleanup { + rm -f $VHU_SOCK0 exit $1 } +function snabbvmx_pid { + pids=$(ps aux | grep snabbvmx | awk '{print $2;}') + for pid in ${pids[@]}; do + if [[ -d "/var/run/snabb/$pid" ]]; then + echo $pid + fi + done +} + trap cleanup EXIT HUP INT QUIT TERM # Import SnabbVMX test_env. @@ -286,7 +297,9 @@ create_mirror_tap_if_needed start_test_env $MIRROR_TAP # Mirror all packets to tap0. -monitor all + +SNABBVMX_PID=$(snabbvmx_pid) +monitor all $SNABBVMX_PID # Run tests. test_ping_to_lwaftr_inet From ab43f22e20cb953f82c7a9cec5ab068d41008910 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Wed, 15 Feb 2017 12:46:00 +0000 Subject: [PATCH 616/631] Delete tap interface before creating it --- src/program/snabbvmx/tests/selftest.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/program/snabbvmx/tests/selftest.sh b/src/program/snabbvmx/tests/selftest.sh index b4fe5425b1..d0feb710ec 100755 --- a/src/program/snabbvmx/tests/selftest.sh +++ b/src/program/snabbvmx/tests/selftest.sh @@ -45,9 +45,11 @@ function tcpreplay { } function create_mirror_tap_if_needed { - sudo ip tuntap add $MIRROR_TAP mode tap &>/dev/null - sudo ip li set dev $MIRROR_TAP up &>/dev/null - sudo ip li sh $MIRROR_TAP &>/dev/null + local TAP_LOG="tap0.log" + sudo ip li delete $MIRROR_TAP &> $TAP_LOG + sudo ip tuntap add $MIRROR_TAP mode tap &>> $TAP_LOG + sudo ip li set dev $MIRROR_TAP up &>> $TAP_LOG + sudo ip li sh $MIRROR_TAP &>> $TAP_LOG if [[ $? -ne 0 ]]; then echo "Couldn't create mirror tap: $MIRROR_TAP" exit 1 From 459e040cb7059933a9be264ee0a43a89c62f3d57 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 28 Mar 2017 11:55:00 +0200 Subject: [PATCH 617/631] Make quickcheck a lwAFTR subprogram --- src/program/lwaftr/README | 1 + src/program/{ => lwaftr}/quickcheck/quickcheck.lua | 4 ++-- src/program/{ => lwaftr}/quickcheck/utils.lua | 0 src/program/lwaftr/tests/propbased/common.lua | 2 +- src/program/lwaftr/tests/propbased/selftest.sh | 6 +++--- 5 files changed, 7 insertions(+), 6 deletions(-) rename src/program/{ => lwaftr}/quickcheck/quickcheck.lua (97%) rename src/program/{ => lwaftr}/quickcheck/utils.lua (100%) diff --git a/src/program/lwaftr/README b/src/program/lwaftr/README index 492096fcd7..56cbaf77f4 100644 --- a/src/program/lwaftr/README +++ b/src/program/lwaftr/README @@ -6,6 +6,7 @@ Usage: snabb lwaftr migrate-configuration snabb lwaftr monitor snabb lwaftr query + snabb lwaftr quickcheck snabb lwaftr run snabb lwaftr run_nohw snabb lwaftr transient diff --git a/src/program/quickcheck/quickcheck.lua b/src/program/lwaftr/quickcheck/quickcheck.lua similarity index 97% rename from src/program/quickcheck/quickcheck.lua rename to src/program/lwaftr/quickcheck/quickcheck.lua index 3dc7ba40ed..751eb448d0 100644 --- a/src/program/quickcheck/quickcheck.lua +++ b/src/program/lwaftr/quickcheck/quickcheck.lua @@ -1,8 +1,8 @@ module(...,package.seeall) -local utils = require('program.quickcheck.utils') +local utils = require('program.lwaftr.quickcheck.utils') -local program_name = 'snabb quickcheck' +local program_name = 'snabb lwaftr quickcheck' local seed, iterations, prop_name, prop_args, prop, prop_info diff --git a/src/program/quickcheck/utils.lua b/src/program/lwaftr/quickcheck/utils.lua similarity index 100% rename from src/program/quickcheck/utils.lua rename to src/program/lwaftr/quickcheck/utils.lua diff --git a/src/program/lwaftr/tests/propbased/common.lua b/src/program/lwaftr/tests/propbased/common.lua index 3422cd6b7f..ed9d5b91d8 100644 --- a/src/program/lwaftr/tests/propbased/common.lua +++ b/src/program/lwaftr/tests/propbased/common.lua @@ -8,7 +8,7 @@ local pci = require("lib.hardware.pci") function make_handle_prop_args(name, duration, pidbox) local handler = function(prop_args) if #prop_args ~= 1 then - print("Usage: snabb quickcheck prop_sameval PCI_ADDR") + print("Usage: snabb lwaftr quickcheck prop_sameval PCI_ADDR") os.exit(1) end diff --git a/src/program/lwaftr/tests/propbased/selftest.sh b/src/program/lwaftr/tests/propbased/selftest.sh index e313cc6206..c4e480cf9a 100755 --- a/src/program/lwaftr/tests/propbased/selftest.sh +++ b/src/program/lwaftr/tests/propbased/selftest.sh @@ -6,6 +6,6 @@ SKIPPED_CODE=43 if [ -z $SNABB_PCI0 ]; then exit $SKIPPED_CODE; fi -./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash $SNABB_PCI0 -./snabb quickcheck program.lwaftr.tests.propbased.prop_nocrash_state $SNABB_PCI0 -./snabb quickcheck program.lwaftr.tests.propbased.prop_sameval $SNABB_PCI0 +./snabb lwaftr quickcheck program.lwaftr.tests.propbased.prop_nocrash $SNABB_PCI0 +./snabb lwaftr quickcheck program.lwaftr.tests.propbased.prop_nocrash_state $SNABB_PCI0 +./snabb lwaftr quickcheck program.lwaftr.tests.propbased.prop_sameval $SNABB_PCI0 From 34bdf3e07a10a1be7168c7d81cbce3c823d6a37b Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 28 Mar 2017 12:02:09 +0200 Subject: [PATCH 618/631] Add quickcheck README --- src/program/lwaftr/quickcheck/README | 14 ++++++++++++++ src/program/lwaftr/quickcheck/README.inc | 1 + 2 files changed, 15 insertions(+) create mode 100644 src/program/lwaftr/quickcheck/README create mode 120000 src/program/lwaftr/quickcheck/README.inc diff --git a/src/program/lwaftr/quickcheck/README b/src/program/lwaftr/quickcheck/README new file mode 100644 index 0000000000..eeeedae192 --- /dev/null +++ b/src/program/lwaftr/quickcheck/README @@ -0,0 +1,14 @@ +Usage: quickcheck [--seed=SEED] [--iterations=ITERATIONS] property_file [property_specific_args] + + -h, --help + Print usage information. + --seed + Sets seed for the tests. + --iterations + Number of runs. + +Utility tool to run property-based tests. + +A property based test is a Lua file which must implement a function called 'property'. +The function must return true in case the checked property was verified or +false otherwise. diff --git a/src/program/lwaftr/quickcheck/README.inc b/src/program/lwaftr/quickcheck/README.inc new file mode 120000 index 0000000000..100b93820a --- /dev/null +++ b/src/program/lwaftr/quickcheck/README.inc @@ -0,0 +1 @@ +README \ No newline at end of file From 76afdf7419d4f80563031a1bffa6c52e8add9fa0 Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 28 Mar 2017 12:04:38 +0200 Subject: [PATCH 619/631] Refactor quickcheck code - Remove references to pflua. - Remove globals. - Use os.exit. - Parse arguments like other lwaftr programs. --- src/program/lwaftr/quickcheck/quickcheck.lua | 119 ++++++++----------- src/program/lwaftr/quickcheck/utils.lua | 2 +- 2 files changed, 53 insertions(+), 68 deletions(-) diff --git a/src/program/lwaftr/quickcheck/quickcheck.lua b/src/program/lwaftr/quickcheck/quickcheck.lua index 751eb448d0..09e1ca8a5b 100644 --- a/src/program/lwaftr/quickcheck/quickcheck.lua +++ b/src/program/lwaftr/quickcheck/quickcheck.lua @@ -1,19 +1,46 @@ module(...,package.seeall) -local utils = require('program.lwaftr.quickcheck.utils') +local lib = require("core.lib") +local utils = require("program.lwaftr.quickcheck.utils") -local program_name = 'snabb lwaftr quickcheck' +local function show_usage(code) + print(require("program.lwaftr.quickcheck.README_inc")) + main.exit(code) +end -local seed, iterations, prop_name, prop_args, prop, prop_info +local function parse_args (args) + local handlers = {} + local opts = { + iterations = 100, + } + function handlers.h() show_usage(0) end + handlers["seed"] = function (arg) + opts["seed"] = assert(tonumber(arg), "seed must be a number") + end + handlers["iterations"] = function (arg) + opts["iterations"] = assert(tonumber(arg), "iterations must be a number") + end + args = lib.dogetopt(args, handlers, "h", + { help="h", ["seed"] = 0, ["iterations"] = 0 }) + if #args == 0 then show_usage(1) end + if not opts.seed then + local seed = math.floor(utils.gmtime() * 1e6) % 10^9 + print("Using time as seed: "..seed) + opts.seed = seed + end + local prop_name = table.remove(args, 1) + + return opts, prop_name, args +end -- Due to limitations of Lua 5.1, finding if a command failed is convoluted. local function find_gitrev() local fd = io.popen('git rev-parse HEAD 2>/dev/null ; echo -n "$?"') local cmdout = fd:read("*all") - fd:close() -- Always true in 5.1, with Lua or LuaJIT + fd:close() -- Always true in 5.1, with Lua or LuaJIT. local _, _, git_ret = cmdout:find("(%d+)$") git_ret = tonumber(git_ret) - if git_ret ~= 0 then -- Probably not in a git repo + if git_ret ~= 0 then -- Probably not in a git repo. return nil else local _, _, sha1 = cmdout:find("(%x+)") @@ -26,68 +53,26 @@ local function print_gitrev_if_available() if rev then print(("Git revision %s"):format(rev)) end end -local function rerun_usage(i) - print(("Rerun as: %s --seed=%s --iterations=%s %s %s"): - format(program_name, seed, i + 1, - prop_name, table.concat(prop_args, " "))) -end - -function initialize(options) - seed, iterations, prop_name, prop_args = - options.seed, options.iterations, options.prop_name, options.prop_args - - if not seed then - seed = math.floor(utils.gmtime() * 1e6) % 10^9 - print("Using time as seed: "..seed) - end - math.randomseed(assert(tonumber(seed))) - - if not iterations then iterations = 100 end - - if not prop_name then - error("No property name specified") - end - - prop = require(prop_name) - if prop.handle_prop_args then - prop_info = prop.handle_prop_args(prop_args) - else - assert(#prop_args == 0, - "Property does not take options "..prop_name) - prop_info = nil - end -end - -function initialize_from_command_line(args) - local options = {} - while #args >= 1 and args[1]:match("^%-%-") do - local arg, _, val = table.remove(args, 1):match("^%-%-([^=]*)(=(.*))$") - assert(arg) - if arg == 'seed' then options.seed = assert(tonumber(val)) - elseif arg == 'iterations' then options.iterations = assert(tonumber(val)) - else error("Unknown argument: " .. arg) end - end - if #args < 1 then - print("Usage: " .. - program_name .. - " [--seed=SEED]" .. - " [--iterations=ITERATIONS]" .. - " property_file [property_specific_args]") - os.exit(1) +local function initialize_property (name, args) + local prop = require(name) + if not prop.handle_prop_args then + assert(#args == 0, "Property does not take options "..name) end - options.prop_name = table.remove(args, 1) - options.prop_args = args - initialize(options) + return prop, prop.handle_prop_args(args) end -function run(args) - initialize_from_command_line(args) - if not prop then - error("Call initialize() or initialize_from_command_line() first") +function run (args) + local opts, prop_name, prop_args = parse_args(args) + local rerun_usage = function (i) + print(("Rerun as: snabb lwaftr quickcheck --seed=%s --iterations=%s %s %s"): + format(program_name, opts.seed, i + 1, + prop_name, table.concat(prop_args, " "))) end + math.randomseed(opts.seed) - for i = 1,iterations do - -- Wrap property and its arguments in a 0-arity function for xpcall + local prop, prop_info = initialize_property(prop_name, prop_args) + for i=1,opts.iterations do + -- Wrap property and its arguments in a 0-arity function for xpcall. local wrap_prop = function() return prop.property(prop_info) end local propgen_ok, expected, got = xpcall(wrap_prop, debug.traceback) if not propgen_ok then @@ -99,14 +84,14 @@ function run(args) end end print("Traceback (this is reliable):") - print(expected) -- This is an error code and traceback in this case + print(expected) -- This is an error code and traceback in this case. rerun_usage(i) - os.exit(1) + main.exit(1) end if not utils.equals(expected, got) then print_gitrev_if_available() print("The property was falsified.") - -- If the property file has extra info available, show it + -- If the property file has extra info available, show it. if prop.print_extra_information then prop.print_extra_information() else @@ -116,10 +101,10 @@ function run(args) utils.pp(got) end rerun_usage(i) - os.exit(1) + main.exit(1) end end - print(iterations.." iterations succeeded.") + print(opts.iterations.." iterations succeeded.") if prop.cleanup then prop.cleanup() end end diff --git a/src/program/lwaftr/quickcheck/utils.lua b/src/program/lwaftr/quickcheck/utils.lua index 69235d4ca3..9cc512c47e 100644 --- a/src/program/lwaftr/quickcheck/utils.lua +++ b/src/program/lwaftr/quickcheck/utils.lua @@ -106,7 +106,7 @@ function choose_with_index(choices) end function selftest () - print("selftest: pf.utils") + print("selftest: quickcheck.utils") local tab = { 1, 2, 3 } assert_equals({ 1, 2, 3, 1, 2, 3 }, concat(tab, tab)) local gu1 = gmtime() From e82413051b7b2eb11068f95321fed9da5948939a Mon Sep 17 00:00:00 2001 From: Diego Pino Garcia Date: Tue, 28 Mar 2017 11:43:45 +0000 Subject: [PATCH 620/631] Add quickcheck unit test --- .../tests/subcommands/quickcheck_test.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/program/lwaftr/tests/subcommands/quickcheck_test.py diff --git a/src/program/lwaftr/tests/subcommands/quickcheck_test.py b/src/program/lwaftr/tests/subcommands/quickcheck_test.py new file mode 100644 index 0000000000..e6a8b951e6 --- /dev/null +++ b/src/program/lwaftr/tests/subcommands/quickcheck_test.py @@ -0,0 +1,23 @@ +""" +Test the "snabb lwaftr quickcheck" subcommand. +""" + +import unittest + +from test_env import SNABB_CMD, BaseTestCase + + +class TestQuickcheck(BaseTestCase): + + cmd_args = [ + str(SNABB_CMD), 'lwaftr', 'quickcheck', '-h' + ] + + def test_run_nohw(self): + output = self.run_cmd(self.cmd_args) + self.assertIn(b'Usage: quickcheck', output, + b'\n'.join((b'OUTPUT', output))) + + +if __name__ == '__main__': + unittest.main() From 084bb44114eda763d6fa321a6222707833d9ba76 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 31 Mar 2017 13:26:16 +0200 Subject: [PATCH 621/631] Add snabb set test and fix get_binding_table This adds another test which checks that erroring occurs on set also not just add (when setting an invlid softwire). It fixes where the initial check of the config happens to prevent it being called too often as well as fixing how often the binding table in the leader is rebuilt. --- src/apps/config/leader.lua | 1 + src/apps/config/support/snabb-softwire-v1.lua | 11 +++--- src/program/lwaftr/bench/bench.lua | 1 + src/program/lwaftr/run/run.lua | 1 + src/program/lwaftr/setup.lua | 5 +-- .../lwaftr/tests/subcommands/config_test.py | 36 +++++++++++++++---- 6 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/apps/config/leader.lua b/src/apps/config/leader.lua index 9c3155b37f..6643e561ff 100644 --- a/src/apps/config/leader.lua +++ b/src/apps/config/leader.lua @@ -69,6 +69,7 @@ function Leader:new (conf) end function Leader:set_initial_configuration (configuration) + self.support.validate_config(configuration) self.current_configuration = configuration self.current_app_graph = self.setup_fn(configuration) self.current_in_place_dependencies = {} diff --git a/src/apps/config/support/snabb-softwire-v1.lua b/src/apps/config/support/snabb-softwire-v1.lua index 9c0ddf59e4..9de7eddcec 100644 --- a/src/apps/config/support/snabb-softwire-v1.lua +++ b/src/apps/config/support/snabb-softwire-v1.lua @@ -18,11 +18,7 @@ local binding_table = require("apps.lwaftr.binding_table") local binding_table_instance local function get_binding_table(conf) - if binding_table_instance ~= nil then - -- TODO: fix me! - binding_table_instance = nil - return get_binding_table(conf) - else + if binding_table_instance == nil then binding_table_instance = binding_table.load(conf) end return binding_table_instance @@ -96,6 +92,11 @@ end local function compute_config_actions(old_graph, new_graph, to_restart, verb, path, arg) + -- If the binding cable changes, remove our cached version. + if path ~= nil and path:match("^/softwire%-config/binding%-table") then + binding_table_instance = nil + end + if verb == 'add' and path == '/softwire-config/binding-table/softwire' then return add_softwire_entry_actions(new_graph, arg) elseif (verb == 'remove' and diff --git a/src/program/lwaftr/bench/bench.lua b/src/program/lwaftr/bench/bench.lua index 4365eea7a4..9ceb8138eb 100644 --- a/src/program/lwaftr/bench/bench.lua +++ b/src/program/lwaftr/bench/bench.lua @@ -57,6 +57,7 @@ function run(args) else setup.apply_scheduling(scheduling) setup.load_bench(graph, conf, inv4_pcap, inv6_pcap, 'sinkv4', 'sinkv6') + setup.validate_config(conf) end app.configure(graph) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index bf266a0d9e..7fdcdcd11e 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -162,6 +162,7 @@ function run(args) else setup.apply_scheduling(scheduling) setup_fn(c, conf, unpack(setup_args)) + setup.validate_config(conf) end engine.configure(c) diff --git a/src/program/lwaftr/setup.lua b/src/program/lwaftr/setup.lua index f4aded241e..9c49f130e1 100644 --- a/src/program/lwaftr/setup.lua +++ b/src/program/lwaftr/setup.lua @@ -24,7 +24,7 @@ local S = require("syscall") local capabilities = {['ietf-softwire']={feature={'binding', 'br'}}} require('lib.yang.schema').set_default_capabilities(capabilities) -local function validate_config(conf) +function validate_config(conf) local support = require("apps.config.support.snabb_softwire_v1") return support.validate_config(conf) end @@ -63,9 +63,6 @@ function lwaftr_app(c, conf) error("One or both of the 'next_hop' values must be specified") end - -- Validate configuration file we've been given. - validate_config(conf) - config.app(c, "reassemblerv4", ipv4_apps.Reassembler, { max_ipv4_reassembly_packets = external_interface.reassembly.max_packets, diff --git a/src/program/lwaftr/tests/subcommands/config_test.py b/src/program/lwaftr/tests/subcommands/config_test.py index 78ed469289..4407835fbb 100644 --- a/src/program/lwaftr/tests/subcommands/config_test.py +++ b/src/program/lwaftr/tests/subcommands/config_test.py @@ -259,30 +259,52 @@ def test_set(self): self.assertEqual(output.strip(), bytes(test_psid, ENC), '\n'.join(('OUTPUT', str(output, ENC)))) - def test_softwire_not_in_psidmap(self): + def test_softwire_not_in_psidmap_add(self): """ - Tests that softwire with a PSID out of PSID map errors + Tests that adding softwire without PSID map entry errors """ # Create a softwire which won't have an IPv4 address in the PSID map. - test_softwire = "{ ipv4 192.168.1.23; psid 72; b4-ipv6 ::1; } " + test_softwire = '{ ipv4 192.168.1.23; psid 72; b4-ipv6 ::1; } ' add_args = self.get_cmd_args('add') add_args.extend(( - "/softwire-config/binding-table/softwire", + '/softwire-config/binding-table/softwire', test_softwire, )) - add_error = "Able to add softwire with without PSID mapping" + add_error = 'Able to add softwire with without PSID mapping' with self.assertRaises(AssertionError, msg=add_error): self.run_cmd(add_args) # Then try and get the softwire added to ensure it's not been added get_args = self.get_cmd_args('get') get_args.append( - "/softwire-config/binding-table/softwire[ipv4=192.168.1.23][psid=72]" + '/softwire-config/binding-table/softwire[ipv4=192.168.1.23][psid=72]' ) - get_error = "Softwire was added with an invalid PSID mapping." + get_error = 'Softwire was added with an invalid PSID mapping.' with self.assertRaises(AssertionError, msg=get_error): self.run_cmd(get_args) + def test_softwire_not_in_psidmap_set(self): + """ + Test setting softwire without PSID map entry errors + """ + # Set the entire binding-table + set_args = self.get_cmd_args('set') + + # PSID mapping for a softwire. + binding_table = ["psid-map { addr 1.1.1.1; psid-length 16;}"] + + # Add a valid softwire + binding_table.append("softwire { ipv4 1.1.1.1; psid 1; b4-ipv6 ::1; }") + + # Add an invalid softwire + binding_table.append("softwire { ipv4 5.5.5.5; psid 1; b4-ipv6 ::1; }") + + set_args.append(" ".join(binding_table)) + err = "Softwire with an invalid PSID mapping was set" + with self.assertRaises(AssertionError, msg=err): + self.run_cmd(set_args) + + if __name__ == '__main__': unittest.main() From 828e67a6d10ac2a1fd648791dab76824d7a1ba9a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 3 Apr 2017 11:51:43 +0200 Subject: [PATCH 622/631] Fix small nit with validate_config call --- src/program/lwaftr/run/run.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/lwaftr/run/run.lua b/src/program/lwaftr/run/run.lua index 7fdcdcd11e..d673c08955 100644 --- a/src/program/lwaftr/run/run.lua +++ b/src/program/lwaftr/run/run.lua @@ -161,8 +161,8 @@ function run(args) setup.reconfigurable(scheduling, setup_fn, c, conf, unpack(setup_args)) else setup.apply_scheduling(scheduling) - setup_fn(c, conf, unpack(setup_args)) setup.validate_config(conf) + setup_fn(c, conf, unpack(setup_args)) end engine.configure(c) From a08f527720c67939154275ce1f8bffba8246ae32 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 10 Apr 2017 14:42:03 +0200 Subject: [PATCH 623/631] Add test to check list with config false doesn't need "key" This is for issue #789 This adds a selftest to verify that a schema with a list that has config false set doesn't error when loaded with data. according to RFC 6020 Section 7.8.2 the "key" statement MAY (but isn't required) when config false is set. --- src/lib/yang/data.lua | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 3586fcaf43..e20b63a762 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -775,5 +775,27 @@ function selftest() assert(parse_uint32('"1"') == 1) assert(parse_uint32(' "1" \n ') == 1) assert(print_uint32(1, string_output_file()) == '1') + + -- Verify that lists can lack keys when "config false;" is set. + local list_wo_key_config_false = [[module config-false-schema { + namespace "urn:ietf:params:xml:ns:yang:config-false-schema"; + prefix "test"; + + container test { + description "Top level node"; + list node { + config false; + description "List without key as config false is set"; + leaf name { type string; } + } + } + }]] + local keyless_schema = schema.load_schema(list_wo_key_config_false) + local keyless_list_schema = load_data_for_schema(keyless_schema, [[ + test { + node { + name "hello"; + } + }]]) print('selfcheck: ok') end From 4385676407e9185bb5e02b957e1a91bfa71dcbc1 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 10 Apr 2017 14:51:50 +0200 Subject: [PATCH 624/631] Fix #789 - Make "keys" optional for list statements with config false This will remove an error which wrongly assumes all list statements have a "key" field. This is only required when there is config false. --- src/lib/yang/data.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index e20b63a762..7b1e596b2d 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -96,7 +96,9 @@ function data_grammar_from_schema(schema) function handlers.list(node) local members=visit_body(node) local keys, values = {}, {} - for k in node.key:split(' +') do keys[k] = assert(members[k]) end + if node.key then + for k in node.key:split(' +') do keys[k] = assert(members[k]) end + end for k,v in pairs(members) do if not keys[k] then values[k] = v end end @@ -791,7 +793,7 @@ function selftest() } }]] local keyless_schema = schema.load_schema(list_wo_key_config_false) - local keyless_list_schema = load_data_for_schema(keyless_schema, [[ + local keyless_list_data = load_data_for_schema(keyless_schema, [[ test { node { name "hello"; From 7753e04843a47b60d8e62ac5afee7216b42309fc Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 20 Apr 2017 13:56:01 +0200 Subject: [PATCH 625/631] Add yang objects to build. * src/Makefile (snabb): Correctly add YANGOBJs. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index ea66c3bccd..b8c439c6f5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -54,7 +54,7 @@ TESTSCRIPTS = $(shell find . -name "selftest.sh" -executable | xargs) PATH := ../lib/luajit/usr/local/bin:$(PATH) -snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(PFLUAASMOBJ) $(INCOBJ) $(LUAJIT_A) +snabb: $(LUAOBJ) $(PFLUAOBJ) $(HOBJ) $(COBJ) $(ARCHOBJ) $(ASMOBJ) $(PFLUAASMOBJ) $(INCOBJ) $(YANGOBJ) $(LUAJIT_A) $(E) "LINK $@" $(Q) $(CC) $(DEBUG) -Wl,--no-as-needed -Wl,-E -Werror -Wall -o $@ $^ \ ../lib/luajit/src/libluajit.a \ From 85d18c7ceaa615f6e586695edcd6d12d9c03ff4f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 20 Apr 2017 14:24:58 +0200 Subject: [PATCH 626/631] Skip snabbwall tests if libndpi isn't available. * src/program/wall/tests/selftest.sh (SKIPPED_CODE): Skip if ndpi isn't available. --- src/program/wall/tests/selftest.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/program/wall/tests/selftest.sh b/src/program/wall/tests/selftest.sh index cf0b55e9f7..e49d0b2c2e 100755 --- a/src/program/wall/tests/selftest.sh +++ b/src/program/wall/tests/selftest.sh @@ -1,5 +1,12 @@ #! /bin/sh set -e + +SKIPPED_CODE=43 +if env LD_PRELOAD=libndpi.so true 2>&1 | grep -qvi error; then + echo "libndpi.so seems to be unavailable; skipping test" + exit $SKIPPED_CODE +fi + mydir=$(dirname "$0") exitcode=0 for path in "${mydir}"/*.test ; do From 9c812cf7045570e05c9979e5698a0304a074b7be Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 20 Apr 2017 14:30:18 +0200 Subject: [PATCH 627/631] Fix libndpi check * src/program/wall/tests/selftest.sh (SKIPPED_CODE): Reverse the sense of the check. --- src/program/wall/tests/selftest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/wall/tests/selftest.sh b/src/program/wall/tests/selftest.sh index e49d0b2c2e..d93f43e9d8 100755 --- a/src/program/wall/tests/selftest.sh +++ b/src/program/wall/tests/selftest.sh @@ -2,7 +2,7 @@ set -e SKIPPED_CODE=43 -if env LD_PRELOAD=libndpi.so true 2>&1 | grep -qvi error; then +if env LD_PRELOAD=libndpi.so true 2>&1 | grep -qi error; then echo "libndpi.so seems to be unavailable; skipping test" exit $SKIPPED_CODE fi From 92bfddbc18c09259a4af3d18b36e38ee487c3048 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 27 Apr 2017 17:11:48 +0200 Subject: [PATCH 628/631] Skip lib.multi_copy selftest if AVX2 unavailable --- src/lib/multi_copy.dasl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lib/multi_copy.dasl b/src/lib/multi_copy.dasl index 7ab2745b80..4e145bee88 100644 --- a/src/lib/multi_copy.dasl +++ b/src/lib/multi_copy.dasl @@ -123,6 +123,14 @@ end function selftest () print("selftest: multi_copy") + + local cpuinfo = require('core.lib').readfile("/proc/cpuinfo", "*a") + assert(cpuinfo, "failed to read /proc/cpuinfo for hardware check") + if not cpuinfo:match("avx2") then + print("selftest: not supported; avx2 unavailable") + return + end + local src = ffi.new('uint8_t[78]', { 1, 2, 2, From 08f3342a95f0ea40030f95392cfbb6fc268e98db Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 2 May 2017 16:51:53 +0200 Subject: [PATCH 629/631] Temporarily disable snabbvmx selftest The snabbvmx selftest currently passes only when run on a system with a 3.x kernel on the host. We will fix this but in the interests of getting the lwaftr in a mergeable state, temporarily disabling this test by renaming it. --- src/program/snabbvmx/tests/{selftest.sh => disabled-selftest.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/program/snabbvmx/tests/{selftest.sh => disabled-selftest.sh} (100%) diff --git a/src/program/snabbvmx/tests/selftest.sh b/src/program/snabbvmx/tests/disabled-selftest.sh similarity index 100% rename from src/program/snabbvmx/tests/selftest.sh rename to src/program/snabbvmx/tests/disabled-selftest.sh From 7c07e80d57d01d59108e6e7698569ad7c574a7aa Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Thu, 11 May 2017 09:49:35 +0200 Subject: [PATCH 630/631] Add soaktest to "snabb lwaftr" usage. --- src/program/lwaftr/README | 1 + 1 file changed, 1 insertion(+) diff --git a/src/program/lwaftr/README b/src/program/lwaftr/README index 56cbaf77f4..781caa093a 100644 --- a/src/program/lwaftr/README +++ b/src/program/lwaftr/README @@ -9,6 +9,7 @@ Usage: snabb lwaftr quickcheck snabb lwaftr run snabb lwaftr run_nohw + snabb lwaftr soaktest snabb lwaftr transient Use --help for per-command usage. From 344e22a0a609d6358a1d535412215936bd198eb3 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 17 May 2017 09:21:50 +0000 Subject: [PATCH 631/631] Disable bus-mastering cleanup for intel_mp See https://github.com/snabbco/snabb/issues/1145 for more discussion. --- src/apps/intel_mp/intel_mp.lua | 2 ++ src/lib/hardware/pci.lua | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/apps/intel_mp/intel_mp.lua b/src/apps/intel_mp/intel_mp.lua index c0a6e470b0..bfea26483b 100644 --- a/src/apps/intel_mp/intel_mp.lua +++ b/src/apps/intel_mp/intel_mp.lua @@ -726,6 +726,7 @@ function Intel1g:init () if not self.master then return end pci.unbind_device_from_linux(self.pciaddress) pci.set_bus_master(self.pciaddress, true) + pci.disable_bus_master_cleanup(self.pciaddress) -- 4.5.3 Initialization Sequence self:disable_interrupts() @@ -865,6 +866,7 @@ function Intel82599:init () if not self.master then return end pci.unbind_device_from_linux(self.pciaddress) pci.set_bus_master(self.pciaddress, true) + pci.disable_bus_master_cleanup(self.pciaddress) for i=1,math.floor(self.linkup_wait/2) do self:disable_interrupts() diff --git a/src/lib/hardware/pci.lua b/src/lib/hardware/pci.lua index 535732093f..b2aa6918a9 100644 --- a/src/lib/hardware/pci.lua +++ b/src/lib/hardware/pci.lua @@ -175,6 +175,12 @@ function set_bus_master (device, enable) f:close() end +-- For devices used by some Snabb apps, PCI bus mastering should +-- outlive the life of the process. +function disable_bus_master_cleanup (device) + shm.unlink('group/dma/pci/'..canonical(device)) +end + -- Shutdown DMA to prevent "dangling" requests for PCI devices opened -- by pid (or other processes in its process group). --