From 7f5f6028462b008e932ca0a81a4da09f41ad2d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Wed, 3 Jul 2019 14:55:06 +0700 Subject: [PATCH 01/23] We will need a schema cache in each schema to handle their `$id` entries. --- include/f5/json/schema.hpp | 6 +++++ src/annotations.cpp | 2 +- src/schema.cache.cpp | 55 +++++++++++++++++++++++--------------- src/schema.cpp | 27 ++++++++++++++++++- 4 files changed, 66 insertions(+), 24 deletions(-) diff --git a/include/f5/json/schema.hpp b/include/f5/json/schema.hpp index 879edab..5d871ee 100644 --- a/include/f5/json/schema.hpp +++ b/include/f5/json/schema.hpp @@ -18,6 +18,9 @@ namespace f5 { namespace json { + class schema_cache; + + /** ## JSON Schema @@ -31,6 +34,9 @@ namespace f5 { public: schema(const fostlib::url &, value v); + /// We need to pre-load this with $id schemas that we find + std::shared_ptr schemas; + const fostlib::url &self() const { return id; } value assertions() const { return validation; } diff --git a/src/annotations.cpp b/src/annotations.cpp index 56eee3b..c178db6 100644 --- a/src/annotations.cpp +++ b/src/annotations.cpp @@ -56,7 +56,7 @@ f5::json::validation::annotations::annotations( spos(std::move(sp)), data(std::move(d)), dpos(std::move(dp)), - schemas{std::make_shared()} { + schemas{s.schemas} { id_handling(this, schemas); definitions(this, base->self(), pointer{}); } diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 14bcec7..89c1a99 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -48,6 +48,30 @@ namespace { static std::map> c; return c; } + auto &g_pre_load() { + static auto cache = []() { + auto cache = std::make_shared(nullptr); + if (const auto p = fostlib::coerce>( + f5::json::c_schema_path.value()); + p) { + const auto fn = fostlib::coerce( + fostlib::string(*p)); + fostlib::json s{ + f5::json::value::parse(fostlib::utf::load_file(fn))}; + cache->insert( + f5::json::schema{fostlib::url{fostlib::url{}, fn}, std::move(s)}); + } else if (f5::json::c_schema_path.value().isarray()) { + for (const auto filepath : f5::json::c_schema_path.value()) { + throw fostlib::exceptions::not_implemented( + __PRETTY_FUNCTION__, + "This type of schema load path not yet supported", + f5::json::c_schema_path.value()); + } + } + return cache; + }(); + return cache; + } } @@ -64,27 +88,12 @@ f5::json::schema_cache::schema_cache(std::shared_ptr b) auto f5::json::schema_cache::root_cache() -> std::shared_ptr { - static std::shared_ptr cache = []() { - auto cache = std::make_shared(nullptr); - if (const auto p = fostlib::coerce>( - c_schema_path.value()); - p) { - const auto fn = fostlib::coerce( - fostlib::string(*p)); - fostlib::json s{ - f5::json::value::parse(fostlib::utf::load_file(fn))}; - cache->insert( - schema{fostlib::url{fostlib::url{}, fn}, std::move(s)}); - } else if (c_schema_path.value().isarray()) { - for (const auto filepath : c_schema_path.value()) { - throw fostlib::exceptions::not_implemented( - "f5::json::schema_cache::root_cache", - "This type of schema load path not yet supported", - c_schema_path.value()); - } - } - return cache; - }(); + /** + * This "special value" for the root cache is pretty bad, but we can't put + * a cache in a schema and load schemas during the root cache creation + * because we end up in an infinite loop. + */ + static std::shared_ptr cache{}; return cache; } @@ -93,7 +102,9 @@ auto f5::json::schema_cache::operator[](f5::u8view u) const -> const schema & { try { const auto pos = cache.find(u); if (pos == cache.end()) { - if (base) { + if (base == root_cache()) { + return (*g_pre_load())[u]; + } else if (base) { return (*base)[u]; } else { std::unique_lock lock{g_loader_cache_mutex()}; diff --git a/src/schema.cpp b/src/schema.cpp index 0868af7..897e1f8 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -6,9 +6,31 @@ */ #include +#include #include +namespace { + /** + * We walk down through all of the keys (excepting some particular values) + * and we register all of the `$id`s that we find. + * + * The naive approach to this is going to be a bit slow, but good enough to + * start with because it will scan each schema we do find twice (once in that + * schema and once in its parent). Multiply nesting schemas increases this + * scan count. + */ + void preload_ids(f5::json::value schema, f5::json::schema_cache&cache) { + if(schema.isobject()) { + for(auto const [key, value] : schema.object()) { + if(value.has_key("$id")) { + } + } + } + } +} + + f5::json::schema::schema(const fostlib::url &b, value v) : id{b, [v]() { @@ -18,7 +40,10 @@ f5::json::schema::schema(const fostlib::url &b, value v) return fostlib::guid(); } }()}, - validation{v} {} + validation{v}, + schemas{std::make_shared()} { + preload_ids(validation, *schemas); +} auto f5::json::schema::validate(value j) const -> validation::result { From f0fb3f940932b385a6c77fcc2e3054de8fa30beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Wed, 3 Jul 2019 17:25:41 +0700 Subject: [PATCH 02/23] clang-format --- src/assertions.numeric.cpp | 8 ++++---- src/schema.cache.cpp | 8 ++++---- src/schema.cpp | 17 ++++++++--------- src/schema.loaders.cpp | 12 ++++++++---- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/assertions.numeric.cpp b/src/assertions.numeric.cpp index 9e200fa..965b4d0 100644 --- a/src/assertions.numeric.cpp +++ b/src/assertions.numeric.cpp @@ -29,13 +29,13 @@ namespace { return p(bound.value(), v) ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](double v) mutable { return p(bound.value(), v) ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](const auto &) mutable { return f5::json::validation::result{std::move(an)}; @@ -46,7 +46,7 @@ namespace { return p(bound.value(), v) ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](double v) mutable { bool passed; @@ -58,7 +58,7 @@ namespace { return passed ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](const auto &) mutable { return f5::json::validation::result{std::move(an)}; diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 89c1a99..4af4093 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -54,12 +54,12 @@ namespace { if (const auto p = fostlib::coerce>( f5::json::c_schema_path.value()); p) { - const auto fn = fostlib::coerce( - fostlib::string(*p)); + const auto fn = + fostlib::coerce(fostlib::string(*p)); fostlib::json s{ f5::json::value::parse(fostlib::utf::load_file(fn))}; - cache->insert( - f5::json::schema{fostlib::url{fostlib::url{}, fn}, std::move(s)}); + cache->insert(f5::json::schema{fostlib::url{fostlib::url{}, fn}, + std::move(s)}); } else if (f5::json::c_schema_path.value().isarray()) { for (const auto filepath : f5::json::c_schema_path.value()) { throw fostlib::exceptions::not_implemented( diff --git a/src/schema.cpp b/src/schema.cpp index 897e1f8..d4aa1dc 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -16,15 +16,14 @@ namespace { * and we register all of the `$id`s that we find. * * The naive approach to this is going to be a bit slow, but good enough to - * start with because it will scan each schema we do find twice (once in that - * schema and once in its parent). Multiply nesting schemas increases this - * scan count. + * start with because it will scan each schema we do find twice (once in + * that schema and once in its parent). Multiply nesting schemas increases + * this scan count. */ - void preload_ids(f5::json::value schema, f5::json::schema_cache&cache) { - if(schema.isobject()) { - for(auto const [key, value] : schema.object()) { - if(value.has_key("$id")) { - } + void preload_ids(f5::json::value schema, f5::json::schema_cache &cache) { + if (schema.isobject()) { + for (auto const [key, value] : schema.object()) { + if (value.has_key("$id")) {} } } } @@ -42,7 +41,7 @@ f5::json::schema::schema(const fostlib::url &b, value v) }()}, validation{v}, schemas{std::make_shared()} { - preload_ids(validation, *schemas); + preload_ids(validation, *schemas); } diff --git a/src/schema.loaders.cpp b/src/schema.loaders.cpp index 4a12221..761c60d 100644 --- a/src/schema.loaders.cpp +++ b/src/schema.loaders.cpp @@ -1,5 +1,6 @@ /** - Copyright 2018-2019, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -23,7 +24,8 @@ const fostlib::setting f5::json::c_schema_loaders( namespace { const fostlib::module c_fost_json_schema{fostlib::c_fost, "json-schema"}; - const fostlib::module c_fost_json_schema_loader{c_fost_json_schema, "loader"}; + const fostlib::module c_fost_json_schema_loader{c_fost_json_schema, + "loader"}; } @@ -96,7 +98,8 @@ namespace { config["base"])}; fostlib::url fetch{ base, url.substr(prefix.code_points())}; - logger("base", base)("fetching", fetch)("found", true); + logger("base", + base)("fetching", fetch)("found", true); try { return http(base, fetch); } catch (fostlib::exceptions::exception &e) { @@ -116,7 +119,8 @@ namespace { } } else { logger("found", false); - logger("reason", "URL doesn't start with supplied prefix"); + logger("reason", + "URL doesn't start with supplied prefix"); } } else { logger("found", false); From 5f75ce492a22885d51248496d8aa16b2186bf75d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Wed, 3 Jul 2019 17:36:32 +0700 Subject: [PATCH 03/23] It's very important that this does not end up as an alias for `nullptr`. --- src/schema.cache.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 4af4093..2c5fd3e 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -92,8 +92,13 @@ auto f5::json::schema_cache::root_cache() -> std::shared_ptr { * This "special value" for the root cache is pretty bad, but we can't put * a cache in a schema and load schemas during the root cache creation * because we end up in an infinite loop. + * + * The returned pointer cannot be `nullptr` or we will go into an infinite + * loop chasing down the pre-load list before we bottom out and try to + * load in more schemas using the schema loading mechanism. */ - static std::shared_ptr cache{}; + static auto cache{ + std::make_shared(std::shared_ptr{})}; return cache; } From 03d1f529c07209857c09baed13f0fd210fd259ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Thu, 4 Jul 2019 10:44:02 +0700 Subject: [PATCH 04/23] Add an initial pass when a schema is first loaded to look for embedded `$id`s --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 5e48f9f..9ec41cc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +2019-07-03 Kirit Saelensminde + Add an initial pass when a schema is first loaded to look for embedded `$id`s so they can be used from anywhere else in the schema. + 2019-01-22 Kirit Saelensminde Add debug logging for the HTTP schema loader. From 74693f2a6056556b9114726c66472bf46fca20d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Thu, 4 Jul 2019 10:44:21 +0700 Subject: [PATCH 05/23] Change the cache map to be keyed on actual URLs --- include/f5/json/schema.cache.hpp | 2 +- src/schema.cache.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/f5/json/schema.cache.hpp b/include/f5/json/schema.cache.hpp index 031f349..4034491 100644 --- a/include/f5/json/schema.cache.hpp +++ b/include/f5/json/schema.cache.hpp @@ -24,7 +24,7 @@ namespace f5 { friend validation::annotations; std::shared_ptr base; - std::map cache; + std::map cache; public: /// Create an empty cache which uses the root cache diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 2c5fd3e..06b3250 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -105,7 +105,7 @@ auto f5::json::schema_cache::root_cache() -> std::shared_ptr { auto f5::json::schema_cache::operator[](f5::u8view u) const -> const schema & { try { - const auto pos = cache.find(u); + const auto pos = cache.find(fostlib::url{u}); if (pos == cache.end()) { if (base == root_cache()) { return (*g_pre_load())[u]; @@ -141,7 +141,9 @@ auto f5::json::schema_cache::operator[](f5::u8view u) const -> const schema & { const fostlib::string cp{std::to_string((int64_t)this)}; fostlib::insert(e.data(), "schema-cache", cp, value::array_t{}); for (const auto &c : cache) { - fostlib::push_back(e.data(), "schema-cache", cp, c.first); + fostlib::push_back( + e.data(), "schema-cache", cp, + fostlib::coerce(c.first)); } throw; } From 8f3fc4ba6c7830350d2e2ffc19c93e3922df6ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Thu, 4 Jul 2019 10:45:33 +0700 Subject: [PATCH 06/23] Preloading requires knowledge of the base URL so we can calculate the real one. --- src/schema.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/schema.cpp b/src/schema.cpp index d4aa1dc..aa94b7c 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -20,10 +20,20 @@ namespace { * that schema and once in its parent). Multiply nesting schemas increases * this scan count. */ - void preload_ids(f5::json::value schema, f5::json::schema_cache &cache) { + void preload_ids( + fostlib::url const &base, + f5::json::value schema, + f5::json::schema_cache &cache) { if (schema.isobject()) { for (auto const [key, value] : schema.object()) { - if (value.has_key("$id")) {} + if (value.has_key("$id") && value["$id"].isatom()) { + auto &s = cache.insert(f5::json::schema{ + fostlib::url{ + base, + fostlib::coerce(value["$id"])}, + value}); + } + preload_ids(base, value, cache); } } } @@ -41,7 +51,7 @@ f5::json::schema::schema(const fostlib::url &b, value v) }()}, validation{v}, schemas{std::make_shared()} { - preload_ids(validation, *schemas); + preload_ids(id, validation, *schemas); } From 9fdda796fe0efd1fa1ff94f9d23338a2fd877b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Thu, 4 Jul 2019 10:46:54 +0700 Subject: [PATCH 07/23] Deal better with URL fragments that aren't JSON pointers. --- src/validator.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/validator.cpp b/src/validator.cpp index 321a64a..ae7e710 100644 --- a/src/validator.cpp +++ b/src/validator.cpp @@ -92,36 +92,40 @@ auto f5::json::validation::first_error(annotations an) -> result { return result{"false", std::move(an.spos), std::move(an.dpos)}; } else if (auto part = an.sroot[an.spos]; part.isobject()) { if (part.has_key("$ref")) { - const auto ref = fostlib::coerce(part["$ref"]); - if (ref.bytes() && *ref.begin() == '#') { + auto const ref = fostlib::coerce(part["$ref"]); + if (ref.starts_with("#/") || ref == "#") { auto valid = first_error( an, - fostlib::jcursor::parse_json_pointer_fragment( - fostlib::coerce(part["$ref"])), + fostlib::jcursor::parse_json_pointer_fragment(ref), an.dpos); if (not valid) return valid; else return annotations(std::move(an), std::move(valid)); } else { + fostlib::string url{ref}; + if (ref.starts_with("#")) { + url = fostlib::coerce( + fostlib::url{an.spos_url(), ref}); + } const auto &cache = *an.schemas; if (const auto frag = - std::find(ref.begin(), ref.end(), '#'); - frag == ref.end()) { - const fostlib::url u{an.spos_url(), ref}; + std::find(url.begin(), url.end(), '#'); + frag == url.end()) { + const fostlib::url u{an.spos_url(), url}; const auto &ref_schema = cache[u.as_string()]; auto valid = first_error(annotations{ an, ref_schema, pointer{}, an.data, an.dpos}); if (not valid) return valid; return annotations{std::move(an), std::move(valid)}; } else { - const f5::u8view us{ref.begin(), frag}; + const f5::u8view us{url.begin(), frag}; const fostlib::url u{an.spos_url(), us}; const auto &ref_schema = cache[u.as_string()]; auto valid = first_error(annotations{ an, ref_schema, fostlib::jcursor::parse_json_pointer_fragment( - f5::u8view{frag, ref.end()}), + f5::u8view{frag, url.end()}), an.data, an.dpos}); if (not valid) return valid; return annotations(std::move(an), std::move(valid)); From 926bfd4ea5f32860b6f3f879dac4c560f276ee9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Thu, 4 Jul 2019 11:05:29 +0700 Subject: [PATCH 08/23] clang-format --- src/assertions.numeric.cpp | 8 ++++---- src/schema.loaders.cpp | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/assertions.numeric.cpp b/src/assertions.numeric.cpp index 9e200fa..965b4d0 100644 --- a/src/assertions.numeric.cpp +++ b/src/assertions.numeric.cpp @@ -29,13 +29,13 @@ namespace { return p(bound.value(), v) ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](double v) mutable { return p(bound.value(), v) ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](const auto &) mutable { return f5::json::validation::result{std::move(an)}; @@ -46,7 +46,7 @@ namespace { return p(bound.value(), v) ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](double v) mutable { bool passed; @@ -58,7 +58,7 @@ namespace { return passed ? f5::json::validation::result{std::move(an)} : f5::json::validation::result{ - rule, an.spos, an.dpos}; + rule, an.spos, an.dpos}; }, [&](const auto &) mutable { return f5::json::validation::result{std::move(an)}; diff --git a/src/schema.loaders.cpp b/src/schema.loaders.cpp index 4a12221..761c60d 100644 --- a/src/schema.loaders.cpp +++ b/src/schema.loaders.cpp @@ -1,5 +1,6 @@ /** - Copyright 2018-2019, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -23,7 +24,8 @@ const fostlib::setting f5::json::c_schema_loaders( namespace { const fostlib::module c_fost_json_schema{fostlib::c_fost, "json-schema"}; - const fostlib::module c_fost_json_schema_loader{c_fost_json_schema, "loader"}; + const fostlib::module c_fost_json_schema_loader{c_fost_json_schema, + "loader"}; } @@ -96,7 +98,8 @@ namespace { config["base"])}; fostlib::url fetch{ base, url.substr(prefix.code_points())}; - logger("base", base)("fetching", fetch)("found", true); + logger("base", + base)("fetching", fetch)("found", true); try { return http(base, fetch); } catch (fostlib::exceptions::exception &e) { @@ -116,7 +119,8 @@ namespace { } } else { logger("found", false); - logger("reason", "URL doesn't start with supplied prefix"); + logger("reason", + "URL doesn't start with supplied prefix"); } } else { logger("found", false); From 1c64ba5da479480d05ebdf6cc9c04f1882a8700e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Thu, 4 Jul 2019 11:05:41 +0700 Subject: [PATCH 09/23] Temporarily lock to and older version of the test suite. --- test/testsuite-v7/testsuite.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/testsuite-v7/testsuite.cpp b/test/testsuite-v7/testsuite.cpp index 7624acb..2d55b76 100644 --- a/test/testsuite-v7/testsuite.cpp +++ b/test/testsuite-v7/testsuite.cpp @@ -18,14 +18,17 @@ namespace { const fostlib::setting c_verbose( __FILE__, "json-schema-testsuite", "Verbose", false, true); - const fostlib::setting - c_base(__FILE__, - "json-schema-testsuite", - "Base URL", - "https://raw.githubusercontent.com/" - "json-schema-org/JSON-Schema-Test-Suite/" - "master/tests/draft7/", - true); + const fostlib::setting c_base( + __FILE__, + "json-schema-testsuite", + "Base URL", + "https://raw.githubusercontent.com/json-schema-org/" + "JSON-Schema-Test-Suite/e64ebf90a001f4e0e18984d2086ea15765cfead2/" + "tests/draft7/", + //"https://raw.githubusercontent.com/" + //"json-schema-org/JSON-Schema-Test-Suite/" + //"master/tests/draft7/", + true); const fostlib::setting c_loaders{ __FILE__, f5::json::c_schema_loaders, []() { From 6ee7fd539f48387034a0adc043c5b0aa4e467ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Mon, 8 Jul 2019 10:00:42 +0700 Subject: [PATCH 10/23] Propogate URL type up call chain. --- include/f5/json/schema.cache.hpp | 5 +++-- src/annotations.cpp | 3 +-- src/schema.cache.cpp | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/f5/json/schema.cache.hpp b/include/f5/json/schema.cache.hpp index 4034491..ebb31dd 100644 --- a/include/f5/json/schema.cache.hpp +++ b/include/f5/json/schema.cache.hpp @@ -1,5 +1,6 @@ /** - Copyright 2018, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -41,7 +42,7 @@ namespace f5 { static std::shared_ptr root_cache(); /// Add a schema at a given position in the cache - const schema &insert(fostlib::string, schema); + const schema &insert(fostlib::url, schema); /// Add a schema at an unnamed position, i.e. only if it /// contains a `$id` describing its proper location const schema &insert(schema); diff --git a/src/annotations.cpp b/src/annotations.cpp index c178db6..4297ca3 100644 --- a/src/annotations.cpp +++ b/src/annotations.cpp @@ -39,8 +39,7 @@ namespace { anp->sroot[anp->spos][sub]["definitions"].object()) { fostlib::url r{anp->base->self(), anp->spos / sub}; const auto &subschema = anp->schemas->insert( - fostlib::string(r.as_string()), - f5::json::schema{base, def.second}); + r, f5::json::schema{base, def.second}); definitions( anp, subschema.self(), sub / "definitions" / def.first); } diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 06b3250..83d9251 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -1,5 +1,6 @@ /** - Copyright 2018, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -162,7 +163,7 @@ auto f5::json::schema_cache::insert(schema s) -> const schema & { } -auto f5::json::schema_cache::insert(fostlib::string n, schema s) +auto f5::json::schema_cache::insert(fostlib::url n, schema s) -> const schema & { cache.insert(std::make_pair(n, s)); return insert(s); From 0ec72a63f68d7e471797eafd036e932e6d8f39ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Mon, 8 Jul 2019 10:02:02 +0700 Subject: [PATCH 11/23] We must rely on the base URL set for the schema in its constructor Because only it knows the correct context for the base URL. --- src/schema.cache.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 83d9251..57b9456 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -152,13 +152,7 @@ auto f5::json::schema_cache::operator[](f5::u8view u) const -> const schema & { auto f5::json::schema_cache::insert(schema s) -> const schema & { - if (s.assertions().has_key("$id")) { - auto parts = fostlib::partition( - fostlib::coerce(s.assertions()["$id"]), "#"); - cache.insert(std::make_pair(parts.first, s)); - } - auto pos = cache.insert( - std::make_pair(fostlib::coerce(s.self()), s)); + auto pos = cache.insert(std::make_pair(s.self(), s)); return pos.first->second; } From 15dde4fd81c1b0900f255af3ff7231e811b2b70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Tue, 9 Jul 2019 12:01:53 +0700 Subject: [PATCH 12/23] Normalise all URLs in schema caches to always have a fragment. --- include/f5/json/schema.cache.hpp | 2 +- src/schema.cache.cpp | 20 +++++++++++--------- src/schema.cpp | 4 +++- src/validator.cpp | 6 ++---- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/include/f5/json/schema.cache.hpp b/include/f5/json/schema.cache.hpp index ebb31dd..0c67591 100644 --- a/include/f5/json/schema.cache.hpp +++ b/include/f5/json/schema.cache.hpp @@ -35,7 +35,7 @@ namespace f5 { schema_cache(std::shared_ptr); /// Perform a lookup in this case and its bases - const schema &operator[](f5::u8view) const; + const schema &operator[](fostlib::url) const; /// The root cache. The root cache is the only cache which /// should have an empty base. diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 57b9456..221c75f 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -46,7 +46,7 @@ namespace { return m; } auto &g_loader_cache() { - static std::map> c; + static std::map> c; return c; } auto &g_pre_load() { @@ -104,24 +104,25 @@ auto f5::json::schema_cache::root_cache() -> std::shared_ptr { } -auto f5::json::schema_cache::operator[](f5::u8view u) const -> const schema & { +auto f5::json::schema_cache::operator[](fostlib::url u) const -> const schema & { try { - const auto pos = cache.find(fostlib::url{u}); + if(not u.fragment()) u.fragment(fostlib::string{}); + const auto pos = cache.find(u); if (pos == cache.end()) { if (base == root_cache()) { - return (*g_pre_load())[u]; + return (*g_pre_load())[std::move(u)]; } else if (base) { - return (*base)[u]; + return (*base)[std::move(u)]; } else { std::unique_lock lock{g_loader_cache_mutex()}; if (auto pos = g_loader_cache().find(u); pos != g_loader_cache().end()) { return *pos->second; } else { - auto l = load_schema(u); + auto l = load_schema(u.as_string()); if (l) { return *( - g_loader_cache()[fostlib::string{u}] = + g_loader_cache()[std::move(u)] = std::move(l)); } else { throw fostlib::exceptions::not_implemented( @@ -136,7 +137,7 @@ auto f5::json::schema_cache::operator[](f5::u8view u) const -> const schema & { if (not e.data().has_key("schema-cache")) { std::unique_lock lock{g_loader_cache_mutex()}; for (const auto &p : g_loader_cache()) { - fostlib::push_back(e.data(), "schema-cache", "", p.first); + fostlib::push_back(e.data(), "schema-cache", "", p.first.as_string()); } } const fostlib::string cp{std::to_string((int64_t)this)}; @@ -159,6 +160,7 @@ auto f5::json::schema_cache::insert(schema s) -> const schema & { auto f5::json::schema_cache::insert(fostlib::url n, schema s) -> const schema & { - cache.insert(std::make_pair(n, s)); + if(not n.fragment()) n.fragment(fostlib::string{}); + cache.insert(std::make_pair(std::move(n), s)); return insert(s); } diff --git a/src/schema.cpp b/src/schema.cpp index aa94b7c..3afbc66 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -1,5 +1,6 @@ /** - Copyright 2018, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -51,6 +52,7 @@ f5::json::schema::schema(const fostlib::url &b, value v) }()}, validation{v}, schemas{std::make_shared()} { + if(not id.fragment()) id.fragment(fostlib::string{}); preload_ids(id, validation, *schemas); } diff --git a/src/validator.cpp b/src/validator.cpp index ae7e710..9c9ad24 100644 --- a/src/validator.cpp +++ b/src/validator.cpp @@ -112,16 +112,14 @@ auto f5::json::validation::first_error(annotations an) -> result { if (const auto frag = std::find(url.begin(), url.end(), '#'); frag == url.end()) { - const fostlib::url u{an.spos_url(), url}; - const auto &ref_schema = cache[u.as_string()]; + const auto &ref_schema = cache[fostlib::url{an.spos_url(), url}]; auto valid = first_error(annotations{ an, ref_schema, pointer{}, an.data, an.dpos}); if (not valid) return valid; return annotations{std::move(an), std::move(valid)}; } else { const f5::u8view us{url.begin(), frag}; - const fostlib::url u{an.spos_url(), us}; - const auto &ref_schema = cache[u.as_string()]; + const auto &ref_schema = cache[fostlib::url{an.spos_url(), us}]; auto valid = first_error(annotations{ an, ref_schema, fostlib::jcursor::parse_json_pointer_fragment( From 423466524ed7f8a5c82bc38e55df3ee7f3929955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Tue, 9 Jul 2019 12:17:47 +0700 Subject: [PATCH 13/23] Deprecate cache lookups by `f5::u8view` --- include/f5/json/schema.cache.hpp | 2 ++ src/schema.cache.cpp | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/f5/json/schema.cache.hpp b/include/f5/json/schema.cache.hpp index 0c67591..88a3cfc 100644 --- a/include/f5/json/schema.cache.hpp +++ b/include/f5/json/schema.cache.hpp @@ -36,6 +36,8 @@ namespace f5 { /// Perform a lookup in this case and its bases const schema &operator[](fostlib::url) const; + [[deprecated("Only call with a fostlib::url")]] + const schema &operator[](f5::u8view) const; /// The root cache. The root cache is the only cache which /// should have an empty base. diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 221c75f..0b2cbd3 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -104,7 +104,11 @@ auto f5::json::schema_cache::root_cache() -> std::shared_ptr { } -auto f5::json::schema_cache::operator[](fostlib::url u) const -> const schema & { +auto f5::json::schema_cache::operator[](f5::u8view u) const -> schema const & { + return (*this)[fostlib::url{u}]; +} + + try { if(not u.fragment()) u.fragment(fostlib::string{}); const auto pos = cache.find(u); From d1d8e89b19d6033071108ad658feb4eec606005a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Tue, 9 Jul 2019 12:19:13 +0700 Subject: [PATCH 14/23] Always strip the fragment when loading a schema. --- src/schema.loaders.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/schema.loaders.cpp b/src/schema.loaders.cpp index 761c60d..3864227 100644 --- a/src/schema.loaders.cpp +++ b/src/schema.loaders.cpp @@ -51,6 +51,7 @@ f5::json::schema_loader::schema_loader(lstring n, schema_loader_fn f) std::unique_ptr f5::json::load_schema(u8view url) { + url = f5::u8view{url.begin(), std::find(url.begin(), url.end(), '#')}; for (const auto loader : c_schema_loaders.value()) { auto fn = g_loaders.find(fostlib::coerce(loader["loader"])); if (fn) { From e5ea8ac283b37363ad8a8619f5343f72ba0c627f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Tue, 9 Jul 2019 12:20:46 +0700 Subject: [PATCH 15/23] clang-format --- include/f5/json/schema.cache.hpp | 4 ++-- src/schema.cache.cpp | 13 +++++++------ src/schema.cpp | 2 +- src/validator.cpp | 6 ++++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/include/f5/json/schema.cache.hpp b/include/f5/json/schema.cache.hpp index 88a3cfc..b58585d 100644 --- a/include/f5/json/schema.cache.hpp +++ b/include/f5/json/schema.cache.hpp @@ -36,8 +36,8 @@ namespace f5 { /// Perform a lookup in this case and its bases const schema &operator[](fostlib::url) const; - [[deprecated("Only call with a fostlib::url")]] - const schema &operator[](f5::u8view) const; + [[deprecated("Only call with a fostlib::url")]] const schema & + operator[](f5::u8view) const; /// The root cache. The root cache is the only cache which /// should have an empty base. diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 0b2cbd3..6ae618d 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -109,8 +109,10 @@ auto f5::json::schema_cache::operator[](f5::u8view u) const -> schema const & { } +auto f5::json::schema_cache::operator[](fostlib::url u) const + -> schema const & { try { - if(not u.fragment()) u.fragment(fostlib::string{}); + if (not u.fragment()) u.fragment(fostlib::string{}); const auto pos = cache.find(u); if (pos == cache.end()) { if (base == root_cache()) { @@ -125,9 +127,7 @@ auto f5::json::schema_cache::operator[](f5::u8view u) const -> schema const & { } else { auto l = load_schema(u.as_string()); if (l) { - return *( - g_loader_cache()[std::move(u)] = - std::move(l)); + return *(g_loader_cache()[u] = std::move(l)); } else { throw fostlib::exceptions::not_implemented( __PRETTY_FUNCTION__, "Schema not found", u); @@ -141,7 +141,8 @@ auto f5::json::schema_cache::operator[](f5::u8view u) const -> schema const & { if (not e.data().has_key("schema-cache")) { std::unique_lock lock{g_loader_cache_mutex()}; for (const auto &p : g_loader_cache()) { - fostlib::push_back(e.data(), "schema-cache", "", p.first.as_string()); + fostlib::push_back( + e.data(), "schema-cache", "", p.first.as_string()); } } const fostlib::string cp{std::to_string((int64_t)this)}; @@ -164,7 +165,7 @@ auto f5::json::schema_cache::insert(schema s) -> const schema & { auto f5::json::schema_cache::insert(fostlib::url n, schema s) -> const schema & { - if(not n.fragment()) n.fragment(fostlib::string{}); + if (not n.fragment()) n.fragment(fostlib::string{}); cache.insert(std::make_pair(std::move(n), s)); return insert(s); } diff --git a/src/schema.cpp b/src/schema.cpp index 3afbc66..b823991 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -52,7 +52,7 @@ f5::json::schema::schema(const fostlib::url &b, value v) }()}, validation{v}, schemas{std::make_shared()} { - if(not id.fragment()) id.fragment(fostlib::string{}); + if (not id.fragment()) id.fragment(fostlib::string{}); preload_ids(id, validation, *schemas); } diff --git a/src/validator.cpp b/src/validator.cpp index 9c9ad24..0a2c47d 100644 --- a/src/validator.cpp +++ b/src/validator.cpp @@ -112,14 +112,16 @@ auto f5::json::validation::first_error(annotations an) -> result { if (const auto frag = std::find(url.begin(), url.end(), '#'); frag == url.end()) { - const auto &ref_schema = cache[fostlib::url{an.spos_url(), url}]; + const auto &ref_schema = + cache[fostlib::url{an.spos_url(), url}]; auto valid = first_error(annotations{ an, ref_schema, pointer{}, an.data, an.dpos}); if (not valid) return valid; return annotations{std::move(an), std::move(valid)}; } else { const f5::u8view us{url.begin(), frag}; - const auto &ref_schema = cache[fostlib::url{an.spos_url(), us}]; + const auto &ref_schema = + cache[fostlib::url{an.spos_url(), us}]; auto valid = first_error(annotations{ an, ref_schema, fostlib::jcursor::parse_json_pointer_fragment( From d826c9a4ffa505ebc30d62a960b8b95e194eae66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Wed, 10 Jul 2019 10:21:30 +0700 Subject: [PATCH 16/23] Properly preserve the original URL. --- src/schema.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema.cpp b/src/schema.cpp index b823991..58226f3 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -47,7 +47,7 @@ f5::json::schema::schema(const fostlib::url &b, value v) if (v.has_key("$id")) { return fostlib::coerce(v["$id"]); } else { - return fostlib::guid(); + return fostlib::string{}; } }()}, validation{v}, From e68ac41b300f301716dda827d6c31b76809cce66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Tue, 16 Jul 2019 17:09:00 +0700 Subject: [PATCH 17/23] Refactor the cache lookup to return the sub-schema part for json pointers. --- include/f5/json/schema.cache.hpp | 9 +++- src/schema.cache.cpp | 71 +++++++++++++++++++++----------- src/validator.cpp | 4 +- 3 files changed, 57 insertions(+), 27 deletions(-) diff --git a/include/f5/json/schema.cache.hpp b/include/f5/json/schema.cache.hpp index b58585d..d2b8003 100644 --- a/include/f5/json/schema.cache.hpp +++ b/include/f5/json/schema.cache.hpp @@ -35,8 +35,9 @@ namespace f5 { schema_cache(std::shared_ptr); /// Perform a lookup in this case and its bases - const schema &operator[](fostlib::url) const; - [[deprecated("Only call with a fostlib::url")]] const schema & + std::pair + operator[](fostlib::url) const; + [[deprecated("Only call with a fostlib::url")]] schema const & operator[](f5::u8view) const; /// The root cache. The root cache is the only cache which @@ -48,6 +49,10 @@ namespace f5 { /// Add a schema at an unnamed position, i.e. only if it /// contains a `$id` describing its proper location const schema &insert(schema); + + private: + /// Looks for an exact match + schema const *recursive_lookup(fostlib::url const &) const; }; diff --git a/src/schema.cache.cpp b/src/schema.cache.cpp index 6ae618d..a9675d9 100644 --- a/src/schema.cache.cpp +++ b/src/schema.cache.cpp @@ -104,38 +104,63 @@ auto f5::json::schema_cache::root_cache() -> std::shared_ptr { } -auto f5::json::schema_cache::operator[](f5::u8view u) const -> schema const & { - return (*this)[fostlib::url{u}]; +auto f5::json::schema_cache::recursive_lookup(fostlib::url const &u) const + -> schema const * { + const auto pos = cache.find(u); + if (pos == cache.end()) { + if (base == root_cache()) { + return (*g_pre_load()).recursive_lookup(u); + } else if (base) { + return (*base).recursive_lookup(u); + } else { + std::unique_lock lock{g_loader_cache_mutex()}; + if (auto pos = g_loader_cache().find(u); + pos != g_loader_cache().end()) { + return pos->second.get(); + } else { + auto l = load_schema(u.as_string()); + if (l) { + return (g_loader_cache()[u] = std::move(l)).get(); + } else { + return nullptr; + } + } + } + } else { + return &pos->second; + } +} + + +auto f5::json::schema_cache::operator[](f5::u8view u) const -> const schema & { + return (*this)[fostlib::url{u}].first; } auto f5::json::schema_cache::operator[](fostlib::url u) const - -> schema const & { + -> std::pair { try { if (not u.fragment()) u.fragment(fostlib::string{}); - const auto pos = cache.find(u); - if (pos == cache.end()) { - if (base == root_cache()) { - return (*g_pre_load())[std::move(u)]; - } else if (base) { - return (*base)[std::move(u)]; - } else { - std::unique_lock lock{g_loader_cache_mutex()}; - if (auto pos = g_loader_cache().find(u); - pos != g_loader_cache().end()) { - return *pos->second; + auto const *s = recursive_lookup(u); + if (s != nullptr) { + return {*s, fostlib::jcursor{}}; + } else { + if (f5::u8view{u.fragment().value()}.starts_with("/")) { + auto sptr = u.fragment().value(); + u.fragment(fostlib::string{}); + s = recursive_lookup(u); + if (s == nullptr) { + throw fostlib::exceptions::not_implemented( + __PRETTY_FUNCTION__, "Schema not found", u); } else { - auto l = load_schema(u.as_string()); - if (l) { - return *(g_loader_cache()[u] = std::move(l)); - } else { - throw fostlib::exceptions::not_implemented( - __PRETTY_FUNCTION__, "Schema not found", u); - } + sptr = fostlib::ascii_printable_string{"#" + sptr}; + return {*s, + fostlib::jcursor::parse_json_pointer_fragment(sptr)}; } + } else { + throw fostlib::exceptions::not_implemented( + __PRETTY_FUNCTION__, "Schema not found", u); } - } else { - return pos->second; } } catch (fostlib::exceptions::exception &e) { if (not e.data().has_key("schema-cache")) { diff --git a/src/validator.cpp b/src/validator.cpp index 0a2c47d..95bb18e 100644 --- a/src/validator.cpp +++ b/src/validator.cpp @@ -112,7 +112,7 @@ auto f5::json::validation::first_error(annotations an) -> result { if (const auto frag = std::find(url.begin(), url.end(), '#'); frag == url.end()) { - const auto &ref_schema = + auto const &[ref_schema, location] = cache[fostlib::url{an.spos_url(), url}]; auto valid = first_error(annotations{ an, ref_schema, pointer{}, an.data, an.dpos}); @@ -120,7 +120,7 @@ auto f5::json::validation::first_error(annotations an) -> result { return annotations{std::move(an), std::move(valid)}; } else { const f5::u8view us{url.begin(), frag}; - const auto &ref_schema = + auto const &[ref_schema, location] = cache[fostlib::url{an.spos_url(), us}]; auto valid = first_error(annotations{ an, ref_schema, From d02cb29c5bbd32a3a2edc87812b020922aa3bac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Thu, 18 Jul 2019 09:22:53 +0700 Subject: [PATCH 18/23] Remove some memory allocations in regex pattern matching. --- src/assertions.string.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/assertions.string.cpp b/src/assertions.string.cpp index 02f3965..bf0cd2e 100644 --- a/src/assertions.string.cpp +++ b/src/assertions.string.cpp @@ -48,8 +48,8 @@ const f5::json::assertion::checker f5::json::assertion::pattern_checker = auto string = fostlib::coerce>( an.data[an.dpos]); if (not string) return validation::result{std::move(an)}; - std::regex re{static_cast( - fostlib::coerce(part))}; + auto const rgx = fostlib::coerce(part); + std::regex re{rgx.data(), rgx.memory().size()}; if (std::regex_search( string->data(), string->data() + string->bytes(), re)) { return validation::result{std::move(an)}; From e44618cb799df7488b619547fbf1ffb8b56d70cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Sun, 21 Jul 2019 16:42:58 +0700 Subject: [PATCH 19/23] Make all URLs relative to the same base. --- test/testsuite-v7/testsuite.cpp | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/test/testsuite-v7/testsuite.cpp b/test/testsuite-v7/testsuite.cpp index 2d55b76..fa89ee0 100644 --- a/test/testsuite-v7/testsuite.cpp +++ b/test/testsuite-v7/testsuite.cpp @@ -1,5 +1,5 @@ /** - Copyright 2018, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. Distributed under the Boost Software License, Version 1.0. See @@ -15,20 +15,19 @@ namespace { + constexpr f5::u8view base_url = + "https://raw.githubusercontent.com/json-schema-org/" + "JSON-Schema-Test-Suite/e64ebf90a001f4e0e18984d2086ea15765cfead2/"; + const fostlib::setting c_verbose( __FILE__, "json-schema-testsuite", "Verbose", false, true); - const fostlib::setting c_base( - __FILE__, - "json-schema-testsuite", - "Base URL", - "https://raw.githubusercontent.com/json-schema-org/" - "JSON-Schema-Test-Suite/e64ebf90a001f4e0e18984d2086ea15765cfead2/" - "tests/draft7/", - //"https://raw.githubusercontent.com/" - //"json-schema-org/JSON-Schema-Test-Suite/" - //"master/tests/draft7/", - true); + const fostlib::setting + c_base(__FILE__, + "json-schema-testsuite", + "Base URL", + base_url + "tests/draft7/", + true); const fostlib::setting c_loaders{ __FILE__, f5::json::c_schema_loaders, []() { @@ -37,10 +36,7 @@ namespace { f5::json::value::object_t github; github["loader"] = "http"; github["prefix"] = "http://localhost:1234/"; - github["base"] = - "https://raw.githubusercontent.com/" - "json-schema-org/JSON-Schema-Test-Suite/master/" - "remotes/"; + github["base"] = base_url + "remotes/"; return github; }()); return loaders; From 567db5910b92fbc7029542e8a060bb3e20ff8451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Mon, 22 Jul 2019 11:28:21 +0700 Subject: [PATCH 20/23] Print some timing information with verbose output. --- json-schema-validator/valid.cpp | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/json-schema-validator/valid.cpp b/json-schema-validator/valid.cpp index df0dd20..4720f5d 100644 --- a/json-schema-validator/valid.cpp +++ b/json-schema-validator/valid.cpp @@ -1,5 +1,6 @@ /** - Copyright 2018, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -9,6 +10,7 @@ #include #include +#include #include @@ -41,8 +43,7 @@ namespace { std::cout << "Assertion: " << e.assertion << "\nSchema position: " << e.spos << "\nData position: " << e.dpos - << "\nSchema: " << s.assertions()[e.spos] - << "\nData: " << d[e.dpos] << std::endl; + << "\nData at that position: " << d[e.dpos] << std::endl; } } @@ -53,23 +54,42 @@ FSL_MAIN("json-schema-validator", "JSON Schema Validator") args.commandSwitch("v", c_verbose); args.commandSwitch("-schema", c_schema); - const f5::json::schema s{fostlib::url{}, load_json(c_schema.value())}; + fostlib::timer time; + + if (c_verbose.value()) { + std::cout << 0.0 << " Loading schema JSON " << c_schema.value() + << std::endl; + } + f5::json::value parsed = load_json(c_schema.value()); + if (c_verbose.value()) { + std::cout << time.seconds() << " Establishing as schema" << std::endl; + } + f5::json::schema const s{fostlib::url{}, parsed}; for (const auto &arg : args) { if (c_verbose.value()) { - std::cout << "Loading and validating " << arg << std::endl; + std::cout << time.seconds() << "Loading " << arg << std::endl; } const auto j = load_json(arg); + if (c_verbose.value()) { + std::cout << time.seconds() << " Validating " << std::endl; + } + auto v = s.validate(j); + if (c_verbose.value()) std::cout << time.seconds() << " Results in\n"; if (c_check_invalid.value()) { - if (const auto v = s.validate(j); v) { + if (v) { std::cout << arg << " validated when it should not have" << std::endl; return 2; } } else { - if (auto v = s.validate(j); not v) { + if (not v) { + if (c_verbose.value()) + std::cout << time.seconds() << " Results in\n"; std::cout << arg << " did not validate" << std::endl; - print(s, j, (f5::json::validation::result::error)std::move(v)); + print(s, j, + static_cast( + std::move(v))); return 1; } } From f153981797ab510b09ea0487031f9e8859bdba7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Mon, 22 Jul 2019 12:14:31 +0700 Subject: [PATCH 21/23] Fix up the build system to parallel run the test suite tests. --- test/testsuite-v7/CMakeLists.txt | 29 +++++++++++++++++++++-------- test/testsuite-v7/testsuite.cpp | 20 ++++++++++++++++++-- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/test/testsuite-v7/CMakeLists.txt b/test/testsuite-v7/CMakeLists.txt index de8794b..5f32cea 100644 --- a/test/testsuite-v7/CMakeLists.txt +++ b/test/testsuite-v7/CMakeLists.txt @@ -1,16 +1,23 @@ if(TARGET stress) set_property(TARGET stress PROPERTY EXCLUDE_FROM_ALL TRUE) - add_executable(json-schema-testsuite EXCLUDE_FROM_ALL testsuite.cpp) - target_link_libraries(json-schema-testsuite f5-json-schema fost-cli fost-inet) - add_dependencies(stress json-schema-testsuite) + add_executable(json-schema-testsuite-runner EXCLUDE_FROM_ALL testsuite.cpp) + target_link_libraries(json-schema-testsuite-runner f5-json-schema fost-cli fost-inet) + add_dependencies(stress json-schema-testsuite-runner) + set(json-schema-testsuite-parts) - function(draftv7 name) - add_custom_command(TARGET json-schema-testsuite - COMMAND json-schema-testsuite -b false + ## The use of a macro rather than a function gets around all sorts of + ## weird cmake stuff to do with the scoping rules of appending to the + ## `json-schema-testsuite-parts` list. + macro(draftv7 name) + add_custom_command(OUTPUT json-schema-testsuite-${name} + COMMAND json-schema-testsuite-runner -b false + "-o" json-schema-testsuite-${name} "-p" ${CMAKE_CURRENT_SOURCE_DIR}/../checks/json-schema.schema.json - ${name}.json) - endfunction() + ${name}.json + MAIN_DEPENDENCY json-schema-testsuite-runner) + list(APPEND json-schema-testsuite-parts json-schema-testsuite-${name}) + endmacro() draftv7(additionalItems) draftv7(additionalProperties) @@ -47,4 +54,10 @@ if(TARGET stress) draftv7(required) draftv7(type) draftv7(uniqueItems) + + + add_custom_target(json-schema-testsuite + DEPENDS ${json-schema-testsuite-parts}) + set_property(TARGET json-schema-testsuite PROPERTY EXCLUDE_FROM_ALL TRUE) + add_dependencies(stress json-schema-testsuite) endif() diff --git a/test/testsuite-v7/testsuite.cpp b/test/testsuite-v7/testsuite.cpp index fa89ee0..732e26d 100644 --- a/test/testsuite-v7/testsuite.cpp +++ b/test/testsuite-v7/testsuite.cpp @@ -1,5 +1,6 @@ /** - Copyright 2018-2019, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -15,12 +16,20 @@ namespace { + + constexpr f5::u8view base_url = "https://raw.githubusercontent.com/json-schema-org/" "JSON-Schema-Test-Suite/e64ebf90a001f4e0e18984d2086ea15765cfead2/"; const fostlib::setting c_verbose( __FILE__, "json-schema-testsuite", "Verbose", false, true); + const fostlib::setting> c_output( + __FILE__, + "json-schema-testsuite", + "Output file", + fostlib::null, + true); const fostlib::setting c_base(__FILE__, @@ -49,6 +58,7 @@ namespace { FSL_MAIN("json-schema-testsuite", "JSON Schema Test Suite Runner") (fostlib::ostream &out, fostlib::arguments &args) { args.commandSwitch("v", c_verbose); + args.commandSwitch("o", c_output); args.commandSwitch("p", f5::json::c_schema_path); fostlib::stringstream buffer; @@ -87,7 +97,13 @@ FSL_MAIN("json-schema-testsuite", "JSON Schema Test Suite Runner") } } - if (failed && not c_verbose.value()) out << buffer.str(); + if (failed && not c_verbose.value()) { + out << buffer.str(); + } else if (not failed && c_output.value()) { + fostlib::utf::save_file( + fostlib::coerce(c_output.value().value()), + ""); + } return std::min(failed, 255); } catch (...) { From e372d866371359fd9ef630dd84092a890e02637a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Mon, 22 Jul 2019 13:24:18 +0700 Subject: [PATCH 22/23] Add in the new behaviour that we expect to be able to use. --- src/validator.cpp | 107 +++++++++++++++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/src/validator.cpp b/src/validator.cpp index 95bb18e..657fcb1 100644 --- a/src/validator.cpp +++ b/src/validator.cpp @@ -1,5 +1,6 @@ /** - Copyright 2018, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. + Distributed under the Boost Software License, Version 1.0. See @@ -11,6 +12,9 @@ #include +#include + + namespace { @@ -92,45 +96,78 @@ auto f5::json::validation::first_error(annotations an) -> result { return result{"false", std::move(an.spos), std::move(an.dpos)}; } else if (auto part = an.sroot[an.spos]; part.isobject()) { if (part.has_key("$ref")) { - auto const ref = fostlib::coerce(part["$ref"]); - if (ref.starts_with("#/") || ref == "#") { - auto valid = first_error( - an, - fostlib::jcursor::parse_json_pointer_fragment(ref), - an.dpos); - if (not valid) - return valid; - else - return annotations(std::move(an), std::move(valid)); + /// Easily switch between the two implementations here for now + bool const old_implementation{true}; + if constexpr (old_implementation) { + auto const ref = fostlib::coerce(part["$ref"]); + if (ref.starts_with("#/") || ref == "#") { + auto valid = first_error( + an, + fostlib::jcursor::parse_json_pointer_fragment( + ref), + an.dpos); + if (not valid) + return valid; + else + return annotations(std::move(an), std::move(valid)); + } else { + fostlib::string url{ref}; + if (ref.starts_with("#")) { + url = fostlib::coerce( + fostlib::url{an.spos_url(), ref}); + } + const auto &cache = *an.schemas; + if (const auto frag = + std::find(url.begin(), url.end(), '#'); + frag == url.end()) { + auto const &[ref_schema, location] = + cache[fostlib::url{an.spos_url(), url}]; + auto valid = first_error( + annotations{an, ref_schema, pointer{}, + an.data, an.dpos}); + if (not valid) return valid; + return annotations{std::move(an), std::move(valid)}; + } else { + const f5::u8view us{url.begin(), frag}; + auto const &[ref_schema, location] = + cache[fostlib::url{an.spos_url(), us}]; + auto valid = first_error(annotations{ + an, ref_schema, + fostlib::jcursor::parse_json_pointer_fragment( + f5::u8view{frag, url.end()}), + an.data, an.dpos}); + if (not valid) return valid; + return annotations(std::move(an), std::move(valid)); + } + } + } else { - fostlib::string url{ref}; - if (ref.starts_with("#")) { - url = fostlib::coerce( - fostlib::url{an.spos_url(), ref}); + + if (part["$ref"].isobject()) { + throw fostlib::exceptions::not_implemented( + __PRETTY_FUNCTION__, "Wait what?!"); } + std::cout << "\nResolving $ref " << part["$ref"]; + std::cout << "Base URL " << an.spos_url() << std::endl; + std::cout << "As u8view " + << fostlib::coerce(part["$ref"]) + << std::endl; + fostlib::url const ref{ + an.spos_url(), + fostlib::coerce(part["$ref"])}; + std::cout << "Gives URL of " << ref << std::endl; const auto &cache = *an.schemas; - if (const auto frag = - std::find(url.begin(), url.end(), '#'); - frag == url.end()) { - auto const &[ref_schema, location] = - cache[fostlib::url{an.spos_url(), url}]; - auto valid = first_error(annotations{ - an, ref_schema, pointer{}, an.data, an.dpos}); - if (not valid) return valid; - return annotations{std::move(an), std::move(valid)}; - } else { - const f5::u8view us{url.begin(), frag}; - auto const &[ref_schema, location] = - cache[fostlib::url{an.spos_url(), us}]; - auto valid = first_error(annotations{ - an, ref_schema, - fostlib::jcursor::parse_json_pointer_fragment( - f5::u8view{frag, url.end()}), - an.data, an.dpos}); - if (not valid) return valid; + auto const &[ref_schema, location] = cache[ref]; + std::cout << ref_schema.self() << " at " << location + << std::endl; + auto valid = first_error(annotations{ + an, ref_schema, location, an.data, an.dpos}); + if (not valid) + return valid; + else return annotations(std::move(an), std::move(valid)); - } } + } else { for (const auto &rule : part.object()) { const auto apos = g_assertions.find(rule.first); From a892962ad60158e917f390d88df858a46ef5b873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirit=20S=C3=A6lensminde?= Date: Mon, 22 Jul 2019 17:35:55 +0700 Subject: [PATCH 23/23] We need to remove the schema cache from the annotation code. This no longer passes even the basic tests. --- include/f5/json/validator.hpp | 5 +++-- src/annotations.cpp | 39 +++++++++++++++-------------------- src/schema.cpp | 12 +++++++---- src/validator.cpp | 5 +++-- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/include/f5/json/validator.hpp b/include/f5/json/validator.hpp index 944951c..6e2fa8a 100644 --- a/include/f5/json/validator.hpp +++ b/include/f5/json/validator.hpp @@ -49,8 +49,6 @@ namespace f5 { value data; pointer dpos; - std::shared_ptr schemas; - private: friend class json::schema; /// Construct the initial location @@ -78,6 +76,9 @@ namespace f5 { /// on the local $id found in parent lexical scopes of the /// JSON fostlib::url spos_url() const; + + /// Return the schema cache for the current schema + schema_cache const &schemas() const; }; diff --git a/src/annotations.cpp b/src/annotations.cpp index 4297ca3..2245869 100644 --- a/src/annotations.cpp +++ b/src/annotations.cpp @@ -1,5 +1,5 @@ /** - Copyright 2018, Proteus Technologies Co Ltd. + Copyright 2018-2019, Proteus Technologies Co Ltd. Distributed under the Boost Software License, Version 1.0. See @@ -22,9 +22,9 @@ namespace { std::shared_ptr schemas) { if (anp->sroot[anp->spos].has_key("$id")) { if (not schemas) { - schemas = - std::make_shared(anp->schemas); - anp->schemas = schemas; +// schemas = +// std::make_shared(anp->schemas); +// anp->schemas = schemas; } anp->base = &schemas->insert( f5::json::schema{anp->base->self(), anp->sroot[anp->spos]}); @@ -38,10 +38,10 @@ namespace { for (const auto &def : anp->sroot[anp->spos][sub]["definitions"].object()) { fostlib::url r{anp->base->self(), anp->spos / sub}; - const auto &subschema = anp->schemas->insert( - r, f5::json::schema{base, def.second}); - definitions( - anp, subschema.self(), sub / "definitions" / def.first); +// const auto &subschema = anp->schemas->insert( +// r, f5::json::schema{base, def.second}); +// definitions( +// anp, subschema.self(), sub / "definitions" / def.first); } } } @@ -54,10 +54,7 @@ f5::json::validation::annotations::annotations( sroot(s.assertions()), spos(std::move(sp)), data(std::move(d)), - dpos(std::move(dp)), - schemas{s.schemas} { - id_handling(this, schemas); - definitions(this, base->self(), pointer{}); + dpos(std::move(dp)) { } @@ -67,10 +64,7 @@ f5::json::validation::annotations::annotations( sroot(s.assertions()), spos(std::move(sp)), data(std::move(d)), - dpos(std::move(dp)), - schemas{std::make_shared(an.schemas)} { - id_handling(this, schemas); - definitions(this, base->self(), pointer{}); + dpos(std::move(dp)) { } @@ -80,9 +74,7 @@ f5::json::validation::annotations::annotations( sroot(an.sroot), spos(std::move(sp)), data(an.data), - dpos(std::move(dp)), - schemas(an.schemas) { - id_handling(this, nullptr); + dpos(std::move(dp)) { } @@ -91,9 +83,7 @@ f5::json::validation::annotations::annotations(annotations &&b, result &&w) sroot{std::move(b.sroot)}, spos{std::move(b.spos)}, data{std::move(b.data)}, - dpos{std::move(b.dpos)}, - schemas{b.schemas} { - id_handling(this, nullptr); + dpos{std::move(b.dpos)} { merge(std::move(w)); } @@ -126,3 +116,8 @@ fostlib::url f5::json::validation::annotations::spos_url() const { } return u; } + + +auto f5::json::validation::annotations::schemas() const -> schema_cache const & { + return *base->schemas; +} diff --git a/src/schema.cpp b/src/schema.cpp index 58226f3..6c4b2a6 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -11,6 +11,9 @@ #include +#include + + namespace { /** * We walk down through all of the keys (excepting some particular values) @@ -28,11 +31,12 @@ namespace { if (schema.isobject()) { for (auto const [key, value] : schema.object()) { if (value.has_key("$id") && value["$id"].isatom()) { + fostlib::url const subschema{base, + fostlib::coerce(value["$id"])}; + std::cout << "Found sub-schema " << subschema << " in " + << base << '\n'; auto &s = cache.insert(f5::json::schema{ - fostlib::url{ - base, - fostlib::coerce(value["$id"])}, - value}); + subschema, value}); } preload_ids(base, value, cache); } diff --git a/src/validator.cpp b/src/validator.cpp index 657fcb1..c1fe20f 100644 --- a/src/validator.cpp +++ b/src/validator.cpp @@ -99,6 +99,7 @@ auto f5::json::validation::first_error(annotations an) -> result { /// Easily switch between the two implementations here for now bool const old_implementation{true}; if constexpr (old_implementation) { + auto const ref = fostlib::coerce(part["$ref"]); if (ref.starts_with("#/") || ref == "#") { auto valid = first_error( @@ -116,7 +117,7 @@ auto f5::json::validation::first_error(annotations an) -> result { url = fostlib::coerce( fostlib::url{an.spos_url(), ref}); } - const auto &cache = *an.schemas; + const auto &cache = an.schemas(); if (const auto frag = std::find(url.begin(), url.end(), '#'); frag == url.end()) { @@ -156,7 +157,7 @@ auto f5::json::validation::first_error(annotations an) -> result { an.spos_url(), fostlib::coerce(part["$ref"])}; std::cout << "Gives URL of " << ref << std::endl; - const auto &cache = *an.schemas; + const auto &cache = an.schemas(); auto const &[ref_schema, location] = cache[ref]; std::cout << ref_schema.self() << " at " << location << std::endl;