Commit 28b1b98b authored by Angus Lees's avatar Angus Lees Committed by GitHub
Browse files

Merge pull request #38 from tomwilkie/37-update-jsonnet-cgo

Update jsonnet-cgo to 04f8990
parents 99caa5bc 556527d0
......@@ -44,11 +44,12 @@ enum ASTType {
AST_IMPORT,
AST_IMPORTSTR,
AST_INDEX,
AST_LOCAL,
AST_IN_SUPER,
AST_LITERAL_BOOLEAN,
AST_LITERAL_NULL,
AST_LITERAL_NUMBER,
AST_LITERAL_STRING,
AST_LOCAL,
AST_OBJECT,
AST_OBJECT_COMPREHENSION,
AST_OBJECT_COMPREHENSION_SIMPLE,
......@@ -61,8 +62,8 @@ enum ASTType {
/** Represents a variable / parameter / field name. */
struct Identifier {
String name;
Identifier(const String &name)
UString name;
Identifier(const UString &name)
: name(name)
{ }
};
......@@ -244,6 +245,7 @@ enum BinaryOp {
BOP_GREATER_EQ,
BOP_LESS,
BOP_LESS_EQ,
BOP_IN,
BOP_MANIFEST_EQUAL,
BOP_MANIFEST_UNEQUAL,
......@@ -273,6 +275,7 @@ static inline std::string bop_string (BinaryOp bop)
case BOP_GREATER_EQ: return ">=";
case BOP_LESS: return "<";
case BOP_LESS_EQ: return "<=";
case BOP_IN: return "in";
case BOP_MANIFEST_EQUAL: return "==";
case BOP_MANIFEST_UNEQUAL: return "!=";
......@@ -472,12 +475,12 @@ struct LiteralNumber : public AST {
/** Represents JSON strings. */
struct LiteralString : public AST {
String value;
UString value;
enum TokenKind { SINGLE, DOUBLE, BLOCK, VERBATIM_SINGLE, VERBATIM_DOUBLE };
TokenKind tokenKind;
std::string blockIndent; // Only contains ' ' and '\t'.
std::string blockTermIndent; // Only contains ' ' and '\t'.
LiteralString(const LocationRange &lr, const Fodder &open_fodder, const String &value,
LiteralString(const LocationRange &lr, const Fodder &open_fodder, const UString &value,
TokenKind token_kind, const std::string &block_indent,
const std::string &block_term_indent)
: AST(lr, AST_LITERAL_STRING, open_fodder), value(value), tokenKind(token_kind),
......@@ -661,7 +664,8 @@ struct ObjectComprehensionSimple : public AST {
AST *array;
ObjectComprehensionSimple(const LocationRange &lr, AST *field, AST *value,
const Identifier *id, AST *array)
: AST(lr, AST_OBJECT_COMPREHENSION_SIMPLE, Fodder{}), field(field), value(value), id(id), array(array)
: AST(lr, AST_OBJECT_COMPREHENSION_SIMPLE, Fodder{}), field(field), value(value), id(id),
array(array)
{ }
};
......@@ -699,6 +703,19 @@ struct SuperIndex : public AST {
{ }
};
/** Represents the e in super construct.
*/
struct InSuper : public AST {
AST *element;
Fodder inFodder;
Fodder superFodder;
InSuper(const LocationRange &lr, const Fodder &open_fodder,
AST *element, const Fodder &in_fodder, const Fodder &super_fodder)
: AST(lr, AST_IN_SUPER, open_fodder), element(element),
inFodder(in_fodder), superFodder(super_fodder)
{ }
};
enum UnaryOp {
UOP_NOT,
UOP_BITWISE_NOT,
......@@ -741,7 +758,7 @@ struct Var : public AST {
/** Allocates ASTs on demand, frees them in its destructor.
*/
class Allocator {
std::map<String, const Identifier*> internedIdentifiers;
std::map<UString, const Identifier*> internedIdentifiers;
ASTs allocated;
public:
template <class T, class... Args> T* make(Args&&... args)
......@@ -760,7 +777,7 @@ class Allocator {
*
* The location used in the Identifier AST is that of the first one parsed.
*/
const Identifier *makeIdentifier(const String &name)
const Identifier *makeIdentifier(const UString &name)
{
auto it = internedIdentifiers.find(name);
if (it != internedIdentifiers.end()) {
......@@ -812,6 +829,7 @@ std::map<BinaryOp, int> build_precedence_map(void)
r[BOP_GREATER_EQ] = 8;
r[BOP_LESS] = 8;
r[BOP_LESS_EQ] = 8;
r[BOP_IN] = 8;
r[BOP_MANIFEST_EQUAL] = 9;
r[BOP_MANIFEST_UNEQUAL] = 9;
......@@ -857,6 +875,7 @@ std::map<std::string, BinaryOp> build_binary_map(void)
r[">="] = BOP_GREATER_EQ;
r["<"] = BOP_LESS;
r["<="] = BOP_LESS_EQ;
r["in"] = BOP_IN;
r["=="] = BOP_MANIFEST_EQUAL;
r["!="] = BOP_MANIFEST_UNEQUAL;
......
......@@ -28,8 +28,8 @@ static const Fodder EF; // Empty fodder.
static const LocationRange E; // Empty.
struct BuiltinDecl {
String name;
std::vector<String> params;
UString name;
std::vector<UString> params;
};
static unsigned long max_builtin = 26;
......@@ -91,13 +91,16 @@ class Desugarer {
return alloc->make<T>(std::forward<Args>(args)...);
}
const Identifier *id(const String &s)
AST *clone(AST *ast)
{ return clone_ast(*alloc, ast); }
const Identifier *id(const UString &s)
{ return alloc->makeIdentifier(s); }
LiteralString *str(const String &s)
LiteralString *str(const UString &s)
{ return make<LiteralString>(E, EF, s, LiteralString::DOUBLE, "", ""); }
LiteralString *str(const LocationRange &loc, const String &s)
LiteralString *str(const LocationRange &loc, const UString &s)
{ return make<LiteralString>(loc, EF, s, LiteralString::DOUBLE, "", ""); }
LiteralNull *null(void)
......@@ -126,7 +129,7 @@ class Desugarer {
false, EF);
}
Apply *stdFunc(const String &name, AST *v)
Apply *stdFunc(const UString &name, AST *v)
{
return make<Apply>(
v->location,
......@@ -141,7 +144,7 @@ class Desugarer {
);
}
Apply *stdFunc(const LocationRange &loc, const String &name, AST *a, AST *b)
Apply *stdFunc(const LocationRange &loc, const UString &name, AST *a, AST *b)
{
return make<Apply>(
loc,
......@@ -181,7 +184,7 @@ class Desugarer {
return make<Error>(msg->location, EF, msg);
}
Error *error(const LocationRange &loc, const String &msg)
Error *error(const LocationRange &loc, const UString &msg)
{
return error(str(loc, msg));
}
......@@ -289,6 +292,10 @@ class Desugarer {
}
}
/** Replaces all occurrences of self, super[f] and e in super with variables.
*
* Returns all variables and original expressions via super_vars.
*/
class SubstituteSelfSuper : public CompilerPass {
Desugarer *desugarer;
SuperVars &superVars;
......@@ -309,8 +316,8 @@ class Desugarer {
}
expr = alloc.make<Var>(expr->location, expr->openFodder, newSelf);
} else if (auto *super_index = dynamic_cast<SuperIndex*>(expr)) {
StringStream ss;
ss << "$outer_super" << (counter++);
UStringStream ss;
ss << U"$outer_super_index" << (counter++);
const Identifier *super_var = desugarer->id(ss.str());
AST *index = super_index->index;
// Desugaring of expr should already have occurred.
......@@ -318,6 +325,13 @@ class Desugarer {
// Re-use super_index since we're replacing it here.
superVars.emplace_back(super_var, super_index);
expr = alloc.make<Var>(expr->location, expr->openFodder, super_var);
} else if (auto *in_super = dynamic_cast<InSuper*>(expr)) {
UStringStream ss;
ss << U"$outer_in_super" << (counter++);
const Identifier *in_super_var = desugarer->id(ss.str());
// Re-use in_super since we're replacing it here.
superVars.emplace_back(in_super_var, in_super);
expr = alloc.make<Var>(expr->location, expr->openFodder, in_super_var);
}
CompilerPass::visitExpr(expr);
}
......@@ -331,13 +345,24 @@ class Desugarer {
if (!field.superSugar) continue;
// We have to bind self/super from expr1 outside the class, as we copy the expression
// into the field body.
AST *index = field.expr1;
// Clone it so that we maintain the AST as a tree.
ClonePass(*alloc).expr(index);
AST *index = clone(field.expr1);
// This will remove self/super.
SubstituteSelfSuper(this, super_vars, counter).expr(index);
AST *super_f = make<SuperIndex>(field.expr1->location, EF, EF, index, EF, nullptr);
field.expr2 = make<Binary>(ast->location, EF, super_f, EF, BOP_PLUS, field.expr2);
field.expr2 = make<Conditional>(
ast->location,
EF,
make<InSuper>(ast->location, EF, index, EF, EF),
EF,
make<Binary>(
ast->location,
EF,
make<SuperIndex>(ast->location, EF, EF, clone(index), EF, nullptr),
EF,
BOP_PLUS,
field.expr2),
EF,
clone(field.expr2));
field.superSugar = false;
}
......@@ -373,13 +398,13 @@ class Desugarer {
auto *_l = id(U"$l");
std::vector<const Identifier*> _i(n);
for (int i = 0; i < n ; ++i) {
StringStream ss;
UStringStream ss;
ss << U"$i_" << i;
_i[i] = id(ss.str());
}
std::vector<const Identifier*> _aux(n);
for (int i = 0; i < n ; ++i) {
StringStream ss;
UStringStream ss;
ss << U"$aux_" << i;
_aux[i] = id(ss.str());
}
......@@ -595,6 +620,9 @@ class Desugarer {
} else if (dynamic_cast<const Importstr*>(ast_)) {
// Nothing to do.
} else if (auto *ast = dynamic_cast<InSuper*>(ast_)) {
desugar(ast->element, obj_level);
} else if (auto *ast = dynamic_cast<Index*>(ast_)) {
desugar(ast->target, obj_level);
if (ast->isSlice) {
......
......@@ -38,6 +38,8 @@ static AST *left_recursive(AST *ast_)
return ast->left;
if (auto *ast = dynamic_cast<Index*>(ast_))
return ast->target;
if (auto *ast = dynamic_cast<InSuper*>(ast_))
return ast->element;
return nullptr;
}
static const AST *left_recursive(const AST *ast_)
......@@ -382,6 +384,13 @@ class Unparser {
o << "importstr";
unparse(ast->file, true);
} else if (auto *ast = dynamic_cast<const InSuper*>(ast_)) {
unparse(ast->element, true);
fill(ast->inFodder, true, true);
o << "in";
fill(ast->superFodder, true, true);
o << "super";
} else if (auto *ast = dynamic_cast<const Index*>(ast_)) {
unparse(ast->target, space_before);
fill(ast->dotFodder, false, false);
......@@ -643,7 +652,7 @@ class EnforceStringStyle : public FmtPass {
if (lit->tokenKind == LiteralString::BLOCK) return;
if (lit->tokenKind == LiteralString::VERBATIM_DOUBLE) return;
if (lit->tokenKind == LiteralString::VERBATIM_SINGLE) return;
String canonical = jsonnet_string_unescape(lit->location, lit->value);
UString canonical = jsonnet_string_unescape(lit->location, lit->value);
unsigned num_single = 0, num_double = 0;
for (char32_t c : canonical) {
if (c == '\'') num_single++;
......@@ -925,7 +934,7 @@ class PrettyFieldNames : public FmtPass {
public:
PrettyFieldNames(Allocator &alloc, const FmtOpts &opts) : FmtPass(alloc, opts) { }
bool isIdentifier(const String &str) {
bool isIdentifier(const UString &str) {
bool first = true;
for (char32_t c : str) {
if (!first && c >= '0' && c <= '9')
......@@ -1304,7 +1313,8 @@ class FixIndentation {
expr(ast->target, new_indent, space_before);
fill(ast->fodderL, false, false, new_indent.lineUp);
column++; // (
const Fodder &first_fodder = ast->args.size() == 0 ? ast->fodderR : argParamFirstFodder(ast->args[0]);
const Fodder &first_fodder = ast->args.size() == 0
? ast->fodderR : argParamFirstFodder(ast->args[0]);
bool strong_indent = false;
// Need to use strong indent if any of the
// arguments (except the first) are preceded by newlines.
......@@ -1485,6 +1495,13 @@ class FixIndentation {
Indent new_indent = newIndent(open_fodder(ast->file), indent, column + 1);
expr(ast->file, new_indent, true);
} else if (auto *ast = dynamic_cast<InSuper*>(ast_)) {
expr(ast->element, indent, space_before);
fill(ast->inFodder, true, true, indent.lineUp);
column += 2; // in
fill(ast->superFodder, true, true, indent.lineUp);
column += 5; // super
} else if (auto *ast = dynamic_cast<Index*>(ast_)) {
expr(ast->target, indent, space_before);
fill(ast->dotFodder, false, false, indent.lineUp);
......
......@@ -459,7 +459,7 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
data = lex_number(c, filename, begin);
break;
// String literals.
// UString literals.
case '"': {
c++;
for (; ; ++c) {
......@@ -485,7 +485,7 @@ Tokens jsonnet_lex(const std::string &filename, const char *input)
}
break;
// String literals.
// UString literals.
case '\'': {
c++;
for (; ; ++c) {
......
......@@ -144,7 +144,7 @@ struct Token {
*/
std::string stringBlockTermIndent;
String data32(void) { return decode_utf8(data); }
UString data32(void) { return decode_utf8(data); }
LocationRange location;
......
......@@ -583,7 +583,8 @@ static char *jsonnet_evaluate_snippet_aux(JsonnetVm *vm, const char *filename,
return nullptr; // Quiet, compiler.
}
static char *jsonnet_evaluate_file_aux(JsonnetVm *vm, const char *filename, int *error, EvalKind kind)
static char *jsonnet_evaluate_file_aux(JsonnetVm *vm, const char *filename, int *error,
EvalKind kind)
{
std::ifstream f;
f.open(filename);
......
......@@ -25,7 +25,7 @@ limitations under the License.
*/
#define LIB_JSONNET_VERSION "v0.9.3"
#define LIB_JSONNET_VERSION "v0.9.4"
/** Return the version string of the Jsonnet interpreter. Conforms to semantic versioning
......
......@@ -840,6 +840,7 @@ class Parser {
// lhs and let lower levels deal with the operator.
switch (peek().kind) {
// Logical / arithmetic binary operator.
case Token::IN:
case Token::OPERATOR:
if (peek().data == ":") {
// Special case for the colons in assert.
......@@ -953,6 +954,17 @@ class Parser {
Token end = parseObjectRemainder(obj, op);
lhs = alloc->make<ApplyBrace>(span(begin, end), begin_fodder, lhs, obj);
} else if (op.kind == Token::IN) {
if (peek().kind == Token::SUPER) {
Token super = pop();
lhs = alloc->make<InSuper>(
span(begin, super), begin_fodder, lhs, op.fodder, super.fodder);
} else {
AST *rhs = parse(precedence - 1);
lhs = alloc->make<Binary>(
span(begin, rhs), begin_fodder, lhs, op.fodder, bop, rhs);
}
} else {
// Logical / arithmetic binary operator.
assert(op.kind == Token::OPERATOR);
......
......@@ -202,6 +202,11 @@ void CompilerPass::visit(Importstr *ast)
visit(ast->file);
}
void CompilerPass::visit(InSuper *ast)
{
expr(ast->element);
}
void CompilerPass::visit(Index *ast)
{
expr(ast->target);
......@@ -313,6 +318,8 @@ void CompilerPass::visitExpr(AST *&ast_)
visit(ast);
} else if (auto *ast = dynamic_cast<Importstr*>(ast_)) {
visit(ast);
} else if (auto *ast = dynamic_cast<InSuper*>(ast_)) {
visit(ast);
} else if (auto *ast = dynamic_cast<Index*>(ast_)) {
visit(ast);
} else if (auto *ast = dynamic_cast<Local*>(ast_)) {
......@@ -357,6 +364,13 @@ void CompilerPass::file(AST *&body, Fodder &final_fodder)
fodder(final_fodder);
}
/** A pass that clones the AST it is given. */
class ClonePass : public CompilerPass {
public:
ClonePass(Allocator &alloc) : CompilerPass(alloc) { }
virtual void expr(AST *&ast);
};
void ClonePass::expr(AST *&ast_)
{
if (auto *ast = dynamic_cast<Apply*>(ast_)) {
......@@ -385,6 +399,8 @@ void ClonePass::expr(AST *&ast_)
ast_ = alloc.clone(ast);
} else if (auto *ast = dynamic_cast<Importstr*>(ast_)) {
ast_ = alloc.clone(ast);
} else if (auto *ast = dynamic_cast<InSuper*>(ast_)) {
ast_ = alloc.clone(ast);
} else if (auto *ast = dynamic_cast<Index*>(ast_)) {
ast_ = alloc.clone(ast);
} else if (auto *ast = dynamic_cast<Local*>(ast_)) {
......@@ -424,3 +440,10 @@ void ClonePass::expr(AST *&ast_)
CompilerPass::expr(ast_);
}
AST *clone_ast(Allocator &alloc, AST *ast)
{
AST *r = ast;
ClonePass(alloc).expr(r);
return r;
}
......@@ -70,6 +70,8 @@ class CompilerPass {
virtual void visit(Importstr *ast);
virtual void visit(InSuper *ast);
virtual void visit(Index *ast);
virtual void visit(Local *ast);
......@@ -106,10 +108,10 @@ class CompilerPass {
};
/** A pass that clones the AST it is given. */
class ClonePass : public CompilerPass {
public:
ClonePass(Allocator &alloc) : CompilerPass(alloc) { }
virtual void expr(AST *&ast);
};
/** Return an equivalent AST that can be modified without affecting the original.
*
* This is a deep copy.
*/
AST *clone_ast(Allocator &alloc, AST *ast);
#endif
......@@ -259,8 +259,8 @@ struct HeapClosure : public HeapEntity {
/** Stores a simple string on the heap. */
struct HeapString : public HeapEntity {
const String value;
HeapString(const String &value)
const UString value;
HeapString(const UString &value)
: value(value)
{ }
};
......
......@@ -90,6 +90,11 @@ static IdSet static_analysis(AST *ast_, bool in_object, const IdSet &vars)
} else if (dynamic_cast<const Importstr*>(ast_)) {
// Nothing to do.
} else if (auto *ast = dynamic_cast<const InSuper*>(ast_)) {
if (!in_object)
throw StaticError(ast_->location, "Can't use super outside of an object.");
append(r, static_analysis(ast->element, in_object, vars));
} else if (auto *ast = dynamic_cast<const Index*>(ast_)) {
append(r, static_analysis(ast->target, in_object, vars));
append(r, static_analysis(ast->index, in_object, vars));
......
......@@ -19,18 +19,18 @@ limitations under the License.
#include "string_utils.h"
#include "static_error.h"
String jsonnet_string_unparse(const String &str, bool single)
UString jsonnet_string_unparse(const UString &str, bool single)
{
StringStream ss;
UStringStream ss;
ss << (single ? U'\'' : U'\"');
ss << jsonnet_string_escape(str, single);
ss << (single ? U'\'' : U'\"');
return ss.str();
}
String jsonnet_string_escape(const String &str, bool single)
UString jsonnet_string_escape(const UString &str, bool single)
{
StringStream ss;
UStringStream ss;
for (std::size_t i=0 ; i<str.length() ; ++i) {
char32_t c = str[i];
switch (c) {
......@@ -61,9 +61,9 @@ String jsonnet_string_escape(const String &str, bool single)
}
String jsonnet_string_unescape(const LocationRange &loc, const String &s)
UString jsonnet_string_unescape(const LocationRange &loc, const UString &s)
{
String r;
UString r;
const char32_t *s_ptr = s.c_str();
for (const char32_t *c = s_ptr; *c != U'\0' ; ++c) {
switch (*c) {
......
......@@ -20,12 +20,12 @@ limitations under the License.
#include "lexer.h"
/** Unparse the string. */
String jsonnet_string_unparse(const String &str, bool single);
UString jsonnet_string_unparse(const UString &str, bool single);
/** Escape special characters. */
String jsonnet_string_escape(const String &str, bool single);
UString jsonnet_string_escape(const UString &str, bool single);
/** Resolve escape chracters in the string. */
String jsonnet_string_unescape(const LocationRange &loc, const String &s);
UString jsonnet_string_unescape(const LocationRange &loc, const UString &s);
#endif
......@@ -119,25 +119,25 @@ static inline char32_t decode_utf8(const std::string &str, size_t &i)
}
/** A string class capable of holding unicode codepoints. */
typedef std::basic_string<char32_t> String;
typedef std::basic_string<char32_t> UString;
static inline void encode_utf8(const String &s, std::string &r)
static inline void encode_utf8(const UString &s, std::string &r)
{
for (char32_t cp : s)
encode_utf8(cp, r);
}
static inline std::string encode_utf8(const String &s)
static inline std::string encode_utf8(const UString &s)
{
std::string r;
encode_utf8(s, r);
return r;
}
static inline String decode_utf8(const std::string &s)
static inline UString decode_utf8(const std::string &s)
{
String r;
UString r;
for (size_t i = 0; i < s.length(); ++i)
r.push_back(decode_utf8(s, i));