From 274bdf8a3dd088e6d18edbed51d1795278cd45b8 Mon Sep 17 00:00:00 2001 From: Sean Huh Date: Fri, 20 Nov 2026 10:34:22 -0800 Subject: [PATCH] Select fields from and index optionals, and error on `.?` of undeclared fields PiperOrigin-RevId: 1026585328 --- .../cel/verifier/CelAstToZ3Translator.java | 11 + .../cel/verifier/CelZ3OperatorTranslator.java | 232 +++++++----------- .../cel/verifier/CelZ3StructTranslator.java | 70 +++++- .../cel/verifier/CelZ3TranslationContext.java | 8 + .../dev/cel/verifier/TranslatedValue.java | 2 + .../cel/verifier/axioms/OptionalAxioms.java | 60 ----- .../cel/verifier/CelVerifierZ3ImplTest.java | 111 ++++++++- 7 files changed, 275 insertions(+), 219 deletions(-) diff --git a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java index a80d1008d..25496cc8c 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelAstToZ3Translator.java @@ -14,6 +14,8 @@ package dev.cel.verifier; +import static com.google.common.base.Preconditions.checkArgument; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; @@ -439,6 +441,15 @@ private TranslatedValue translateCall(CelExpr expr, CelAbstractSyntaxTree ast) { long exprId = expr.id(); String functionName = call.function(); ImmutableList argsAst = call.args(); + if (functionName.equals(Operator.OPTIONAL_SELECT.getFunction())) { + checkArgument( + !call.target().isPresent() + && argsAst.size() == 2 + && argsAst.get(1).constantOrDefault().getKind() == CelConstant.Kind.STRING_VALUE, + "Malformed AST: operator %s expects an operand and a field name", + Operator.OPTIONAL_SELECT); + return structTranslator.translateSelect(expr, ast); + } List args = new ArrayList<>(); diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java index e48f1d33b..c223dbd9b 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3OperatorTranslator.java @@ -74,10 +74,9 @@ Optional translateFunctionCall( TranslatedValue resultChain = opOpt .map(op -> translateOperatorCall(op, args, ast)) - .orElseGet( - () -> - TranslatedValue.propagateStrict( - ctx, typeSystem, functionName, typeSystem.mkError(), args)); + .orElseGet(() -> + TranslatedValue.propagateStrict( + ctx, typeSystem, functionName, typeSystem.mkError(), args)); CelReference reference = ast.getReferenceMap().get(exprId); CelFunctionDecl decl = functionRegistry.getDeclaration(functionName).orElse(null); @@ -177,15 +176,6 @@ Optional translateFunctionCall( CelZ3Exprs.mkOrFlattened( ctx, currentApprox, ctx.mkAnd(isList, CelZ3Exprs.mkOrFlattened(ctx, taints))); } - } else if (op == Operator.OPTIONAL_SELECT) { - // A chained optional select (e.g. `a.?b.?c`) selects from the value of an optional operand. - Expr operand = z3Args.get(0); - Expr target = - ctx.mkITE( - typeSystem.isOptional(operand), - typeSystem.getOptionalValue(typeSystem.getOptionalRef(operand)), - operand); - currentApprox = CelZ3Exprs.mkOrFlattened(ctx, currentApprox, context.isTruncatedMap(target)); } return Optional.of( @@ -359,9 +349,8 @@ private TranslatedValue translateOperatorCall( return TranslatedValue.propagateStrict( ctx, typeSystem, op.getFunction(), typeSystem.mkError(), args); case INDEX: - return translateIndex(args, ast, false); case OPTIONAL_INDEX: - return translateIndex(args, ast, true); + return translateIndex(args, ast, op); case CONDITIONAL: return translateConditional(args, ast); case NOT_STRICTLY_FALSE: @@ -1033,160 +1022,125 @@ private TranslatedValue translateEquality( .withApproximation(CelZ3Exprs.mkOrFlattened(ctx, taints)); } - private Expr buildListIndex( - Expr lhsTrans, Expr rhsTrans, BoolExpr typeGuard, boolean isOptional) { - Expr listRef = typeSystem.getListRef(lhsTrans); - SeqExpr seq = typeSystem.getSeq(listRef); - Expr index = typeSystem.getInt(rhsTrans); - BoolExpr inBounds = - ctx.mkAnd( - ctx.mkGe((ArithExpr) index, ctx.mkInt(0)), - ctx.mkLt((ArithExpr) index, ctx.mkLength(seq))); - - return finishIndex(inBounds, ctx.mkNth(seq, (ArithExpr) index), typeGuard, isOptional); + /** A list element or map value, read without regard to optionals. */ + private static final class IndexAccess { + private final BoolExpr valid; + private final BoolExpr found; + private final Expr value; + private final BoolExpr isApproximate; + + private IndexAccess(BoolExpr valid, BoolExpr found, Expr value, BoolExpr isApproximate) { + this.valid = valid; + this.found = found; + this.value = value; + this.isApproximate = isApproximate; + } + } + + private IndexAccess listAccess(Expr lhsTrans, Expr rhsTrans) { + SeqExpr seq = typeSystem.getSeq(typeSystem.getListRef(lhsTrans)); + IntExpr index = typeSystem.getInt(rhsTrans); + return new IndexAccess( + ctx.mkAnd(typeSystem.isList(lhsTrans), typeSystem.isInt(rhsTrans)), + ctx.mkAnd(ctx.mkGe(index, ctx.mkInt(0)), ctx.mkLt(index, ctx.mkLength(seq))), + ctx.mkNth(seq, index), + /* isApproximate= */ ctx.mkFalse()); } - private Expr buildMapIndex( - Expr lhsTrans, Expr rhsTrans, BoolExpr typeGuard, boolean isOptional) { + private IndexAccess mapAccess(Expr lhsTrans, Expr rhsTrans) { ImmutableList equalKeys = typeSystem.equalKeys(rhsTrans); - return finishIndex( + return new IndexAccess( + typeSystem.isMap(lhsTrans), typeSystem.hasEqualKey(lhsTrans, equalKeys), typeSystem.valueAtEqualKey(lhsTrans, equalKeys), - typeGuard, - isOptional); + /* isApproximate= */ ctx.mkFalse()); } - // Returns val where found, or else an error (or an empty optional). - private Expr finishIndex(BoolExpr found, Expr val, BoolExpr typeGuard, boolean isOptional) { - context.constrainValidValue(ctx.mkAnd(typeGuard, found), val); - if (isOptional) { - Expr resultOptRef = ctx.mkApp(typeSystem.optionalOfRefFunc(), val); - context.addConstraint(ctx.mkEq(typeSystem.getOptionalValue(resultOptRef), val)); - context.addConstraint(typeSystem.optHasValue(resultOptRef)); - return ctx.mkITE(found, typeSystem.mkOptionalOf(resultOptRef), typeSystem.mkOptionalNone()); + /** + * Looks up {@code rhsTrans} in the list or map {@code lhsTrans}. The value is approximate if the + * static element type isn't guaranteed and it may not conform. + */ + private IndexAccess indexAccess( + Expr lhsTrans, Expr rhsTrans, CelType containerType, boolean isLhsTypeGuaranteed) { + IndexAccess list = listAccess(lhsTrans, rhsTrans); + IndexAccess map = mapAccess(lhsTrans, rhsTrans); + IndexAccess access = + new IndexAccess( + ctx.mkOr(list.valid, map.valid), + (BoolExpr) ctx.mkITE(list.valid, list.found, map.found), + ctx.mkITE(list.valid, list.value, map.value), + /* isApproximate= */ ctx.mkFalse()); + BoolExpr isRead = ctx.mkAnd(access.valid, access.found); + context.constrainValidValue(isRead, access.value); + if (containerType.kind() != CelKind.MAP && containerType.kind() != CelKind.LIST) { + return access; } - return ctx.mkITE(found, val, typeSystem.mkError()); + CelType elemType = + containerType.kind() == CelKind.MAP + ? ((MapType) containerType).valueType() + : ((ListType) containerType).elemType(); + BoolExpr conforms = typeConstraintGenerator.createTypeConstraintForType(access.value, elemType); + if (isLhsTypeGuaranteed) { + context.addConstraint(ctx.mkImplies(isRead, conforms)); + return access; + } + // The type checker may narrow other static types (e.g. `(['a'] + dyn_var)[1]` is typed + // string), so the element may not conform. + return new IndexAccess( + access.valid, + access.found, + access.value, + CelZ3Exprs.mkAndFlattened(ctx, isRead, CelZ3Exprs.mkNotFlattened(ctx, conforms))); } + /** + * Translates {@code lhs[rhs]} or {@code lhs[?rhs]}, per {@code op}. An optional lhs is indexed + * optionally (e.g. {@code optional.of([])[0]} is {@code optional.none()}). + */ private TranslatedValue translateIndex( - List args, CelAbstractSyntaxTree ast, boolean isOptional) { + List args, CelAbstractSyntaxTree ast, Operator op) { TranslatedValue lhs = args.get(0); TranslatedValue rhs = args.get(1); CelType lhsType = extractAstTypeOrDefault(lhs, ast); boolean isLhsTypeGuaranteed = lhsType.equals(guaranteedType(lhs, ast)); + CelType containerType = lhsType instanceof OptionalType ? lhsType.parameters().get(0) : lhsType; Expr lhsTrans = lhs.z3Expr(); Expr rhsTrans = rhs.z3Expr(); + BoolExpr isOptionalLhs = typeSystem.isOptional(lhsTrans); + Expr optRef = typeSystem.getOptionalRef(lhsTrans); + BoolExpr lhsHasValue = typeSystem.optHasValue(optRef); + lhsTrans = ctx.mkITE(isOptionalLhs, typeSystem.getOptionalValue(optRef), lhsTrans); + // An empty optional's value can't be read, so constraining it is harmless. + IndexAccess access = indexAccess(lhsTrans, rhsTrans, containerType, isLhsTypeGuaranteed); + Expr element = + ctx.mkITE(ctx.mkAnd(access.valid, access.found), access.value, typeSystem.mkError()); + Expr optionalElement = + ctx.mkITE( + access.valid, + ctx.mkITE( + access.found, context.mkOptionalOf(access.value), typeSystem.mkOptionalNone()), + typeSystem.mkError()); + Expr actualValue = + ctx.mkITE( + isOptionalLhs, + ctx.mkITE(lhsHasValue, optionalElement, typeSystem.mkOptionalNone()), + op == Operator.OPTIONAL_INDEX ? optionalElement : element); - BoolExpr isLhsOpt = ctx.mkFalse(); - BoolExpr lhsHasValue = ctx.mkFalse(); - BoolExpr shouldEvaluate = ctx.mkTrue(); - - if (isOptional) { - isLhsOpt = typeSystem.isOptional(lhsTrans); - Expr optRef = typeSystem.getOptionalRef(lhsTrans); - lhsHasValue = typeSystem.optHasValue(optRef); - - lhsTrans = ctx.mkITE(isLhsOpt, typeSystem.getOptionalValue(optRef), lhsTrans); - shouldEvaluate = (BoolExpr) ctx.mkITE(isLhsOpt, lhsHasValue, ctx.mkTrue()); - - if (lhsType instanceof OptionalType) { - lhsType = lhsType.parameters().get(0); - } - } - - TranslatedValue element = - buildAndConstrainIndex( - lhsTrans, rhsTrans, lhsType, isLhsTypeGuaranteed, shouldEvaluate, isOptional); - Expr actualValue = element.z3Expr(); - - if (isOptional) { - actualValue = - ctx.mkITE( - ctx.mkAnd(isLhsOpt, ctx.mkNot(lhsHasValue)), - typeSystem.mkOptionalNone(), - actualValue); - } - - String opName = - isOptional ? Operator.OPTIONAL_INDEX.getFunction() : Operator.INDEX.getFunction(); BoolExpr isListIndexTruncated = context.recordUnrollLimitApproximation( ctx.mkAnd( - shouldEvaluate, typeSystem.isList(lhsTrans), typeSystem.isInt(rhsTrans), ctx.mkGe(typeSystem.getInt(rhsTrans), ctx.mkInt(comprehensionUnrollLimit)), ctx.mkLt( typeSystem.getInt(rhsTrans), ctx.mkLength(typeSystem.getSeq(typeSystem.getListRef(lhsTrans)))))); - BoolExpr isMapIndexTruncated = ctx.mkAnd(shouldEvaluate, context.isTruncatedMap(lhsTrans)); - return TranslatedValue.propagateStrict(ctx, typeSystem, opName, actualValue, args) + BoolExpr isMapIndexTruncated = context.isTruncatedMap(lhsTrans); + return TranslatedValue.propagateStrict(ctx, typeSystem, op.getFunction(), actualValue, args) .withApproximation( CelZ3Exprs.mkOrFlattened( - ctx, isListIndexTruncated, isMapIndexTruncated, element.isApproximate())); - } - - /** - * Returns the element of {@code lhsTrans} at {@code rhsTrans}, which is approximate if it may not - * conform to a static element type that is not guaranteed. - */ - private TranslatedValue buildAndConstrainIndex( - Expr lhsTrans, - Expr rhsTrans, - CelType lhsType, - boolean isLhsTypeGuaranteed, - BoolExpr shouldEvaluate, - boolean isOptional) { - CelType expectedElemType = null; - if (lhsType.kind() == CelKind.LIST) { - expectedElemType = ((ListType) lhsType).elemType(); - } else if (lhsType.kind() == CelKind.MAP) { - expectedElemType = ((MapType) lhsType).valueType(); - } - - if (expectedElemType != null) { - Expr actualValue; - if (lhsType.kind() == CelKind.MAP) { - actualValue = buildMapIndex(lhsTrans, rhsTrans, shouldEvaluate, isOptional); - } else { - BoolExpr isIntIndex = typeSystem.isInt(rhsTrans); - actualValue = - ctx.mkITE( - isIntIndex, - buildListIndex( - lhsTrans, rhsTrans, ctx.mkAnd(shouldEvaluate, isIntIndex), isOptional), - typeSystem.mkError()); - } - - CelType finalType = isOptional ? OptionalType.create(expectedElemType) : expectedElemType; - BoolExpr isDefined = ctx.mkAnd(shouldEvaluate, ctx.mkNot(typeSystem.isError(actualValue))); - BoolExpr typeConstraint = - typeConstraintGenerator.createTypeConstraintForType(actualValue, finalType); - if (isLhsTypeGuaranteed) { - context.addConstraint(ctx.mkImplies(isDefined, typeConstraint)); - return TranslatedValue.create(actualValue, typeSystem, ctx.mkFalse()); - } - // The type checker may narrow other static types (e.g. `(['a'] + dyn_var)[1]` is typed - // string), so the element may not conform. - return TranslatedValue.create( - actualValue, - typeSystem, - CelZ3Exprs.mkAndFlattened( - ctx, isDefined, CelZ3Exprs.mkNotFlattened(ctx, typeConstraint))); - } - - BoolExpr isListGuard = - ctx.mkAnd(shouldEvaluate, typeSystem.isList(lhsTrans), typeSystem.isInt(rhsTrans)); - BoolExpr isMapGuard = ctx.mkAnd(shouldEvaluate, typeSystem.isMap(lhsTrans)); - - return TranslatedValue.create( - CelZ3Exprs.SwitchBuilder.newBuilder(ctx) - .addCase(isListGuard, buildListIndex(lhsTrans, rhsTrans, isListGuard, isOptional)) - .addCase(isMapGuard, buildMapIndex(lhsTrans, rhsTrans, isMapGuard, isOptional)) - .build(typeSystem.mkError()), - typeSystem, - ctx.mkFalse()); + ctx, isListIndexTruncated, isMapIndexTruncated, access.isApproximate)); } private TranslatedValue translateConditional( diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java b/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java index 11cc36191..645720443 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3StructTranslator.java @@ -26,6 +26,7 @@ import com.microsoft.z3.SeqExpr; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.ast.CelExpr; +import dev.cel.common.ast.CelExpr.ExprKind; import dev.cel.common.internal.ProtoTimeUtils; import dev.cel.common.types.CelKind; import dev.cel.common.types.CelType; @@ -536,13 +537,36 @@ private FieldAccess getMsgAccess(Expr operand, String field, BoolExpr typeGua return new FieldAccess(presence, value); } + /** + * Translates {@code e.f}, {@code has(e.f)} or {@code e.?f} (an optional of {@code e.f} if {@code + * has(e.f)}, else empty). + */ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { - CelExpr.CelSelect select = celExpr.select(); + boolean isOptional = celExpr.getKind() == ExprKind.Kind.CALL; + CelExpr operandExpr = isOptional ? celExpr.call().args().get(0) : celExpr.select().operand(); + String field = + isOptional + ? celExpr.call().args().get(1).constant().stringValue() + : celExpr.select().field(); + boolean testOnly = !isOptional && celExpr.select().testOnly(); long exprId = celExpr.id(); - TranslatedValue operandTv = translateExpr(select.operand(), ast); + TranslatedValue operandTv = translateExpr(operandExpr, ast); Expr operand = operandTv.z3Expr(); - String field = select.field(); - CelType operandType = extractAstTypeOrDefault(ast, select.operand().id()); + CelType operandType = extractAstTypeOrDefault(ast, operandExpr.id()); + // Performance only: an operand of a known type other than optional isn't one. + boolean mayBeOptional = + operandType instanceof OptionalType + || !CelZ3OperatorTranslator.isStaticallyKnown(operandType); + BoolExpr isOptionalOperand = ctx.mkFalse(); + BoolExpr hasValue = ctx.mkFalse(); + BoolExpr shouldEvaluate = ctx.mkTrue(); + if (mayBeOptional) { + Expr optRef = typeSystem.getOptionalRef(operand); + isOptionalOperand = typeSystem.isOptional(operand); + hasValue = typeSystem.optHasValue(optRef); + shouldEvaluate = (BoolExpr) ctx.mkITE(isOptionalOperand, hasValue, ctx.mkTrue()); + operand = ctx.mkITE(isOptionalOperand, typeSystem.getOptionalValue(optRef), operand); + } Expr presenceResult; Expr valueResult; @@ -550,11 +574,11 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { BoolExpr presenceError = ctx.mkFalse(); if (operandType instanceof MapType) { - FieldAccess mapAcc = getMapAccess(operand, field, ctx.mkTrue()); + FieldAccess mapAcc = getMapAccess(operand, field, shouldEvaluate); presenceResult = mapAcc.presence; valueResult = ctx.mkITE(mapAcc.presence, mapAcc.value, typeSystem.mkError()); } else if (operandType.kind() == CelKind.STRUCT) { - FieldAccess msgAcc = getMsgAccess(operand, field, ctx.mkTrue()); + FieldAccess msgAcc = getMsgAccess(operand, field, shouldEvaluate); presenceResult = msgAcc.presence; Expr defaultVal = getDefaultValueForType(extractAstTypeOrDefault(ast, exprId)); valueResult = ctx.mkITE(msgAcc.presence, msgAcc.value, defaultVal); @@ -563,8 +587,8 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { BoolExpr isMap = typeSystem.isMap(operand); BoolExpr isMessage = typeSystem.isMessage(operand); - FieldAccess mapAcc = getMapAccess(operand, field, isMap); - FieldAccess msgAcc = getMsgAccess(operand, field, isMessage); + FieldAccess mapAcc = getMapAccess(operand, field, ctx.mkAnd(shouldEvaluate, isMap)); + FieldAccess msgAcc = getMsgAccess(operand, field, ctx.mkAnd(shouldEvaluate, isMessage)); presenceResult = CelZ3Exprs.SwitchBuilder.newBuilder(ctx) @@ -590,13 +614,13 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { msgRead.addCase(isType, typeSystem.mkError()); continue; } - if (select.testOnly()) { + if (testOnly) { continue; } CelType fieldType = declared.get().type(); context.addConstraint( ctx.mkImplies( - ctx.mkAnd(isMessage, isType, msgAcc.presence), + ctx.mkAnd(shouldEvaluate, isMessage, isType, msgAcc.presence), typeConstraintGenerator.createTypeConstraintForType(msgAcc.value, fieldType))); msgRead.addCase( isType, ctx.mkITE(msgAcc.presence, msgAcc.value, getDefaultValueForType(fieldType))); @@ -605,7 +629,10 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { unmodeledMessage = context.recordApproximation( "messages of types the type provider doesn't declare", - ctx.mkAnd(isMessage, ctx.mkNot(CelZ3Exprs.mkOrFlattened(ctx, isModeledType)))); + ctx.mkAnd( + shouldEvaluate, + isMessage, + ctx.mkNot(CelZ3Exprs.mkOrFlattened(ctx, isModeledType)))); Expr mapRead = ctx.mkITE(mapAcc.presence, mapAcc.value, typeSystem.mkError()); // Testing an undeclared field, or a field of something other than a map or message, errors. presenceError = @@ -623,9 +650,25 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { } Expr fieldAccess = - select.testOnly() + testOnly ? typeSystem.withRuntimeError(typeSystem.wrapBool(presenceResult), presenceError) : valueResult; + Expr chained = fieldAccess; + if (!testOnly && (isOptional || mayBeOptional)) { + chained = + ctx.mkITE( + presenceError, + typeSystem.mkError(), + ctx.mkITE( + (BoolExpr) presenceResult, + context.mkOptionalOf(valueResult), + typeSystem.mkOptionalNone())); + fieldAccess = isOptional ? chained : valueResult; + } + if (mayBeOptional) { + Expr empty = testOnly ? typeSystem.mkBool(false) : typeSystem.mkOptionalNone(); + fieldAccess = ctx.mkITE(isOptionalOperand, ctx.mkITE(hasValue, chained, empty), fieldAccess); + } context.addConstraint(typeConstraintGenerator.createTypeConstraint(fieldAccess, exprId, ast)); @@ -636,7 +679,8 @@ TranslatedValue translateSelect(CelExpr celExpr, CelAbstractSyntaxTree ast) { return operandType.kind() == CelKind.STRUCT ? result : result.withApproximation( - CelZ3Exprs.mkOrFlattened(ctx, context.isTruncatedMap(operand), unmodeledMessage)); + CelZ3Exprs.mkOrFlattened( + ctx, ctx.mkAnd(shouldEvaluate, context.isTruncatedMap(operand)), unmodeledMessage)); } private static CelType extractAstTypeOrDefault(CelAbstractSyntaxTree ast, long id) { diff --git a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java index fb6f7270d..05e6e307a 100644 --- a/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java +++ b/verifier/src/main/java/dev/cel/verifier/CelZ3TranslationContext.java @@ -80,6 +80,14 @@ void constrainValidValue(BoolExpr guard, Expr value) { addConstraint(ctx.mkImplies(guard, typeSystem.isValidValue(value))); } + /** Returns an optional holding {@code value}. */ + Expr mkOptionalOf(Expr value) { + Expr optRef = ctx.mkApp(typeSystem.optionalOfRefFunc(), value); + addConstraint(ctx.mkEq(typeSystem.getOptionalValue(optRef), value)); + addConstraint(typeSystem.optHasValue(optRef)); + return typeSystem.mkOptionalOf(optRef); + } + void addInputVariable(Expr variable, CelType declaredType) { inputVariableTypes.put(variable, declaredType); } diff --git a/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java b/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java index 7ea52abc8..201739f66 100644 --- a/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java +++ b/verifier/src/main/java/dev/cel/verifier/TranslatedValue.java @@ -199,6 +199,8 @@ private static String extractOpName(CelExpr expr) { switch (expr.exprKind().getKind()) { case SELECT: return "select_" + expr.select().field() + "_" + expr.select().testOnly(); + case CALL: // e.?f + return "optional_select_" + expr.call().args().get(1).constant().stringValue(); case STRUCT: StringBuilder structSb = new StringBuilder("struct_").append(expr.struct().messageName()); for (CelExpr.CelStruct.Entry entry : expr.struct().entries()) { diff --git a/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java b/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java index dd983d36b..748b6f6ca 100644 --- a/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java +++ b/verifier/src/main/java/dev/cel/verifier/axioms/OptionalAxioms.java @@ -20,12 +20,10 @@ import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OF_NON_ZERO_VALUE; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OR; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_OR_VALUE; -import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_SELECT; import static dev.cel.extensions.CelOptionalLibrary.OptionalDeclaration.OPTIONAL_VALUE; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.microsoft.z3.ArrayExpr; import com.microsoft.z3.BoolExpr; import com.microsoft.z3.Context; import com.microsoft.z3.Expr; @@ -33,7 +31,6 @@ import com.microsoft.z3.SeqExpr; import dev.cel.common.CelFunctionDecl; import dev.cel.common.CelOverloadDecl; -import dev.cel.verifier.CelZ3Exprs; import dev.cel.verifier.CelZ3TypeSystem; import java.util.Optional; @@ -88,63 +85,6 @@ final class OptionalAxioms { (ctx, ts, sink, val, other) -> { Expr optRef = ts.getOptionalRef(val); return Optional.of(ctx.mkITE(ts.optHasValue(optRef), val, other)); - }), - createBinaryAxiom( - OPTIONAL_SELECT, - (ctx, ts, sink, operand, field) -> { - Expr optRef = ts.getOptionalRef(operand); - BoolExpr isOpt = ts.isOptional(operand); - BoolExpr hasValue = ts.optHasValue(optRef); - Expr actualOperand = ctx.mkITE(isOpt, ts.getOptionalValue(optRef), operand); - - BoolExpr isMap = ts.isMap(actualOperand); - BoolExpr isMsg = ts.isMessage(actualOperand); - BoolExpr isValidTarget = ctx.mkOr(isMap, isMsg); - - Expr msgFieldZ3Str = ts.getString(field); - Expr mapFieldCelVal = field; - - Expr msgRef = ts.getMessageRef(actualOperand); - Expr mapRef = ts.getMapRef(actualOperand); - - Expr presence = - CelZ3Exprs.SwitchBuilder.newBuilder(ctx) - .addCase( - isMsg, - ctx.mkSelect((ArrayExpr) ts.getMsgPresence(msgRef), msgFieldZ3Str)) - .addCase( - isMap, - ctx.mkSelect((ArrayExpr) ts.getMapPresence(mapRef), mapFieldCelVal)) - .build(ctx.mkFalse()); - - Expr value = - CelZ3Exprs.SwitchBuilder.newBuilder(ctx) - .addCase( - isMsg, ctx.mkSelect((ArrayExpr) ts.getMsgValues(msgRef), msgFieldZ3Str)) - .addCase( - isMap, - ctx.mkSelect((ArrayExpr) ts.getMapValues(mapRef), mapFieldCelVal)) - .build(ts.mkError()); - - BoolExpr valNotError = ctx.mkNot(ctx.mkEq(value, ts.mkError())); - BoolExpr shouldEvaluate = (BoolExpr) ctx.mkITE(isOpt, hasValue, ctx.mkTrue()); - sink.accept( - ctx.mkImplies( - CelZ3Exprs.mkAndFlattened( - ctx, shouldEvaluate, isValidTarget, (BoolExpr) presence), - valNotError)); - - Expr resultOptRef = ctx.mkApp(ts.optionalOfRefFunc(), value); - sink.accept(ctx.mkEq(ts.getOptionalValue(resultOptRef), value)); - sink.accept(ts.optHasValue(resultOptRef)); - - Expr optionalResult = - ctx.mkITE( - (BoolExpr) presence, ts.mkOptionalOf(resultOptRef), ts.mkOptionalNone()); - - Expr result = ctx.mkITE(isValidTarget, optionalResult, ts.mkError()); - return Optional.of( - ctx.mkITE(ctx.mkAnd(isOpt, ctx.mkNot(hasValue)), ts.mkOptionalNone(), result)); })); private static BoolExpr isZeroValue(Context ctx, CelZ3TypeSystem ts, Expr val) { diff --git a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java index 8951801f7..0fd3ba2cb 100644 --- a/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java +++ b/verifier/src/test/java/dev/cel/verifier/CelVerifierZ3ImplTest.java @@ -747,8 +747,6 @@ private enum IsSatisfiableInconclusiveTestCase { "dyn({'a': int_list}) == {'a': [1, 2, 3, 4, 5, dyn('not_an_int')]}"), BEYOND_BMC_LIMIT_DYN_MAP_LITERAL_OPTIONAL_VALUE_EQUALITY( "dyn({?'a': optional.of(int_list)}) == {'a': [1, 2, 3, 4, 5, dyn('not_an_int')]}"), - BEYOND_BMC_LIMIT_OPTIONAL_MAP_LITERAL_EQUALITY_WITH_DYN_MAP( - "size(int_list) == 6 && dyn({?'a': optional.of(int_list)}) == dyn_map"), BEYOND_BMC_LIMIT_EQUALITY_WITH_DYN_LIST( "int_list == dyn_list && size(dyn_list) == 6 && 'not_an_int' in dyn_list"), BEYOND_BMC_LIMIT_IN_DYN_LIST("size(int_list) == 6 && dyn('not_an_int') in dyn(int_list)"), @@ -757,8 +755,6 @@ private enum IsSatisfiableInconclusiveTestCase { BEYOND_BMC_LIMIT_IN_DYN_MAP("size(string_int_map) == 6 && dyn(1) in dyn(string_int_map)"), BEYOND_BMC_LIMIT_IN_DYN_LIST_CONCATENATION("'a' in dyn(int_list + [1])"), // A long map's type constraint is truncated even if only its key or its value type is known. - BEYOND_BMC_LIMIT_DYN_KEY_MAP_IN_LITERAL( - "dyn(dyn_int_map) in [{1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: dyn('not_an_int')}]"), BEYOND_BMC_LIMIT_DYN_VALUE_MAP_IN_LITERAL( "dyn(string_dyn_map) in [{'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, dyn(6): 6}]"), // Map presence is only linked to the keys up to the unroll limit. @@ -793,6 +789,23 @@ public void isSatisfiable_inconclusive(@TestParameter IsSatisfiableInconclusiveT assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); } + @Test + public void isSatisfiable_mapLiteralWithDynMapBeyondBmcLimit_inconclusive( + @TestParameter({ + "size(int_list) == 2 && dyn({?'a': optional.of(int_list)}) == dyn_map", + "dyn(dyn_int_map) in [{1: 1, 2: dyn('not_an_int')}]" + }) + String expr) + throws Exception { + CelAbstractSyntaxTree ast = CEL.compile(expr).getAst(); + // A small unroll limit keeps the solver fast; the maps are still past it. + CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setComprehensionUnrollLimit(1)); + + CelVerificationResult result = verifier.isSatisfiable(ast); + + assertThat(result.status()).isEqualTo(VerificationStatus.INCONCLUSIVE); + } + @Test public void isSatisfiable_maskedByBmcNested_inconclusive() throws Exception { String expr = @@ -1212,6 +1225,37 @@ private enum IsAlwaysTrueTestCase { OPTIONAL_OF_EQ_OF_VAR("optional.of(x) == optional.of(x)"), OPTIONAL_OF_NEQ_OF_VAR("x != y ? optional.of(x) != optional.of(y) : true"), OPTIONAL_OR_VALUE_VAR("optional.of(x).orValue(y) == x"), + // Indexing follows the value, whatever type the checker narrowed it to. + NARROWED_MAP_INDEXED_AS_LIST( + "dyn_var == optional.of([2]) ? dyn_var.orValue({0: 1})[0] == 2 : true"), + NARROWED_LIST_INDEXED_AS_MAP( + "dyn_var == optional.of({0: 2}) ? dyn_var.orValue([1])[0] == 2 : true"), + NARROWED_LIST_GIVEN_OPTIONAL( + "dyn_var == optional.of(optional.of([2]))" + + " ? dyn(dyn_var.orValue([1])[0]) == optional.of(2) : true"), + NARROWED_LIST_GIVEN_OPTIONAL_OPTIONAL_INDEX( + "dyn_var == optional.of(optional.of([2]))" + + " ? dyn(dyn_var.orValue([1])[?0]) == optional.of(2) : true"), + NARROWED_OPTIONAL_INDEX( + "dyn_var == optional.of(['a'])" + + " ? dyn(dyn_var.orValue(optional.of([1]))[0]) == 'a' : true"), + NARROWED_OPTIONAL_OPTIONAL_INDEX( + "dyn_var == optional.of([2])" + + " ? dyn_var.orValue(optional.of([1]))[?0] == optional.of(2) : true"), + NARROWED_OPTIONAL_MAP_INDEX( + "dyn_var == optional.of({'a': 2})" + + " ? dyn(dyn_var.orValue(optional.of({'a': 1}))['a']) == 2 : true"), + // An empty optional gives an empty optional for any key. + OPTIONAL_LIST_INDEXED_BY_STRING("TestAllTypes{}.?repeated_int64[dyn('a')] == optional.none()"), + OPTIONAL_LIST_OPTIONALLY_INDEXED_BY_STRING( + "TestAllTypes{}.?repeated_int64[?dyn('a')] == optional.none()"), + // The checker finds several overloads for an optional(dyn) lhs, so the runtime indexes a value + // typed optional that isn't one by what it is. + NARROWED_OPTIONAL_DYN_INDEX( + "dyn_var == optional.of([2]) ? dyn_var.orValue(optional.of(dyn([1])))[0] == 2 : true"), + NARROWED_OPTIONAL_DYN_OPTIONAL_INDEX( + "dyn_var == optional.of([2]) ? dyn_var.orValue(optional.of(dyn([1])))[?0] == optional.of(2)" + + " : true"), OPTIONAL_OR_VALUE_FALLBACK_VAR("optional.none().orValue(x) == x"), OPTIONAL_OR_VAR("optional.of(x).or(optional.of(y)) == optional.of(x)"), OPTIONAL_OR_FALLBACK_VAR("optional.none().or(optional.of(x)) == optional.of(x)"), @@ -1795,6 +1839,17 @@ private enum UnconditionalErrorTestCase { "dyn(TestAllTypes.NestedMessage{}).single_int64 == 0" + " || dyn(TestAllTypes.NestedMessage{}).single_int64 != 0"), MAP_MISSING_KEY("{'a': 1}['b'] == 0 || !({'a': 1}['b'] == 0)"), + OPTIONAL_SELECT_UNDECLARED_FIELD_OF_DYN_MESSAGE( + "dyn(TestAllTypes{}).?unknown_field.hasValue()" + + " || !dyn(TestAllTypes{}).?unknown_field.hasValue()"), + OPTIONAL_CHAINING_SELECT_UNDECLARED_FIELD( + "optional.of(dyn(TestAllTypes{})).unknown_field.hasValue()" + + " || !optional.of(dyn(TestAllTypes{})).unknown_field.hasValue()"), + // No overload indexes a list with a string, even an empty optional list. + // A present optional's value must be a map or message to select from. + OPTIONAL_CHAINING_SELECT_FROM_NULL( + "{'null_key': dyn(null)}.?null_key.invalid.hasValue()" + + " || !{'null_key': dyn(null)}.?null_key.invalid.hasValue()"), COLLECTION_ERROR("{'a': 1 / 0} == {'a': 1 / 0}"), LIST_ERROR("[1 / 0] == [1 / 0]"), STRICT_LITERAL_ERROR("{'a': 1/0}.exists(k, k == 'a') == {'a': 1/0}.exists(k, k == 'a')"), @@ -1871,7 +1926,12 @@ private enum MalformedAstTestCase { Operator.LESS, ImmutableList.of( CelExpr.newBuilder().setId(2).setConstant(CelConstant.ofValue(1L)).build()), - "expects 2 arguments, got 1"); + "expects 2 arguments, got 1"), + OPTIONAL_SELECT( + Operator.OPTIONAL_SELECT, + ImmutableList.of( + CelExpr.newBuilder().setId(2).setConstant(CelConstant.ofValue(1L)).build()), + "expects an operand and a field name"); final Operator operator; final ImmutableList args; @@ -1933,6 +1993,17 @@ private enum IsAlwaysTrueViolationTestCase { "dyn_var == optional.of(1) ? optional.ofNonZeroValue('').or(dyn_var).value() != dyn(1.0)" + " : true", "dyn_var = optional\\.of\\(1\\)"), + // Indexing 42 errors whether or not the overload is picked by its narrowed type. + NARROWED_OPTIONAL_INDEX_NOT_CONTAINER( + "dyn_var == optional.of(42) ? dyn_var.orValue(optional.of([1]))[0] in [42, optional.of(42)]" + + " : true", + "dyn_var = optional\\.of\\(42\\)"), + // Selecting from an optional follows its value, here 42 rather than the map its type says. + NARROWED_OPTIONAL_MAP_SELECTED( + "dyn_var == optional.of(optional.of(42))" + + " ? (dyn_var.orValue(optional.of({'a': 1})).a.hasValue()" + + " || !dyn_var.orValue(optional.of({'a': 1})).a.hasValue()) : true", + "dyn_var = optional\\.of\\(optional\\.of\\(42\\)\\)"), // Map literal keys must be strings, ints, uints or bools, and can't repeat. MAP_LITERAL_DOUBLE_KEY("{1.0: 'a'}[dyn(1)] == 'a'", "Condition is not always true\\."), MAP_LITERAL_BYTES_KEY("size({b'a': 1}) == 1", "Condition is not always true\\."), @@ -2301,8 +2372,8 @@ public void isAlwaysTrue_dynamicMapComprehensionNestedEquality_violated() throws // A looser timeout than the default: comparing a dyn map with a literal admits every numeric // representation of its keys and values, a state space that takes 5-25 s to explore remotely // (longer on GitHub runners). - // TODO: Restore 20 s once select on optionals is modeled (`request` may be - // `optional.none()`) and the slowdown since map literal key validation is recovered. + // TODO: Restore 20 s once the slowdown since map literal key validation is + // recovered. CelVerifier verifier = withRuntimeOracle(CEL, b -> b.setTimeout(Duration.ofSeconds(60))); CelVerificationResult result = verifier.isAlwaysTrue(ast); @@ -2811,6 +2882,32 @@ private enum EquivalenceTestCase { OPTIONAL_FIELD_SELECTION_MAP_COMPREHENSION( "{'a': 1, 'b': 2}.transformMap(k, v, v > 1, v).?b", "optional.of(2)"), OPTIONAL_FIELD_SELECTION_BINDER("cel.bind(m, {'a': 1}, m.?a)", "optional.of(1)"), + // Selecting from or indexing an optional does so optionally. + OPTIONAL_CHAINING_SELECT( + "optional.of({'c': {'index': 'goodbye'}}).c.index", "optional.of('goodbye')"), + OPTIONAL_CHAINING_SELECT_MISSING("optional.of({'c': {}}).c.missing", "optional.none()"), + OPTIONAL_CHAINING_SELECT_NONE("optional.none().c", "optional.none()"), + OPTIONAL_CHAINING_SELECT_OPTIONAL_VALUE( + "optional.of({'a': optional.of(1)}).a", "optional.of(optional.of(1))"), + OPTIONAL_CHAINING_HAS_PRESENT("has(optional.of({'c': 1}).c)", "true"), + OPTIONAL_CHAINING_HAS_ABSENT("has(optional.of({'c': 1}).d)", "false"), + OPTIONAL_CHAINING_HAS_NONE("has(optional.none().c)", "false"), + OPTIONAL_CHAINING_SELECT_PROTO3_ZERO( + "optional.of(TestAllTypes{single_int64: 0}).single_int64", "optional.none()"), + OPTIONAL_CHAINING_SELECT_UNSET_WRAPPER( + "optional.of(TestAllTypes{}).single_int64_wrapper", "optional.none()"), + OPTIONAL_CHAINING_INDEX("optional.of(['list-value'])[0]", "optional.of('list-value')"), + OPTIONAL_CHAINING_INDEX_MISSING("optional.of({'c': {}}).c['missing']", "optional.none()"), + OPTIONAL_CHAINING_INDEX_NONE("optional.none()[0]", "optional.none()"), + OPTIONAL_CHAINING_INDEX_DYN("dyn(optional.of([1]))[0]", "optional.of(1)"), + OPTIONAL_INDEX_DYN_OPTIONAL("dyn(optional.of([1]))[?0]", "optional.of(1)"), + OPTIONAL_SELECT_THEN_INDEX( + "optional.of(TestAllTypes{repeated_string: ['foo']}).?repeated_string[0]", + "optional.of('foo')"), + OPTIONAL_CHAINING_DYN( + "dyn_var == optional.of({'a': [1]}) ? dyn_var.a[0] : optional.of(1)", "optional.of(1)"), + OPTIONAL_CHAINING_DYN_NONE( + "request == optional.none() ? request.l : optional.none()", "optional.none()"), JSON_VALUE_BOOL("google.protobuf.Value{bool_value: true}", "true"), JSON_VALUE_NUMBER("google.protobuf.Value{number_value: 1.0}", "1.0"), JSON_VALUE_NULL("google.protobuf.Value{null_value: 0}", "null"),