@@ -359,6 +359,7 @@ struct bpf_prog_aux {
u32 id;
u32 func_cnt; /* used by non-func prog as the number of func progs */
u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
+ bool no_verifier_zext; /* No zero extension insertion by verifier. */
bool offload_requested;
struct bpf_prog **func;
void *jit_data; /* JIT specific data. arch dependent */
@@ -7224,6 +7224,38 @@ static int opt_remove_nops(struct bpf_verifier_env *env)
return 0;
}
+static int opt_subreg_zext(struct bpf_verifier_env *env)
+{
+ struct bpf_insn_aux_data *aux = env->insn_aux_data;
+ int i, delta = 0, len = env->prog->len;
+ struct bpf_insn *insns = env->prog->insnsi;
+ struct bpf_insn zext_patch[3];
+ struct bpf_prog *new_prog;
+
+ zext_patch[1] = BPF_ALU64_IMM(BPF_LSH, 0, 32);
+ zext_patch[2] = BPF_ALU64_IMM(BPF_RSH, 0, 32);
+ for (i = 0; i < len; i++) {
+ struct bpf_insn insn;
+
+ if (!aux[i + delta].zext_dst)
+ continue;
+
+ insn = insns[i + delta];
+ zext_patch[0] = insn;
+ zext_patch[1].dst_reg = insn.dst_reg;
+ zext_patch[2].dst_reg = insn.dst_reg;
+ new_prog = bpf_patch_insn_data(env, i + delta, zext_patch, 3);
+ if (!new_prog)
+ return -ENOMEM;
+ env->prog = new_prog;
+ insns = new_prog->insnsi;
+ aux = env->insn_aux_data;
+ delta += 2;
+ }
+
+ return 0;
+}
+
/* convert load instructions that access fields of a context type into a
* sequence of instructions that access fields of the underlying structure:
* struct __sk_buff -> struct sk_buff
@@ -8022,7 +8054,18 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
if (ret == 0)
ret = check_max_stack_depth(env);
- /* instruction rewrites happen after this point */
+ /* Instruction rewrites happen after this point.
+ * For offload target, finalize hook has all aux insn info, do any
+ * customized work there.
+ */
+ if (ret == 0 && bpf_jit_32bit_opt &&
+ !bpf_prog_is_dev_bound(env->prog->aux)) {
+ ret = opt_subreg_zext(env);
+ env->prog->aux->no_verifier_zext = !!ret;
+ } else {
+ env->prog->aux->no_verifier_zext = true;
+ }
+
if (is_priv) {
if (ret == 0)
opt_hard_wire_dead_code_branches(env);