{
 "validation": [
  {
   "design": "D01",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 63,
   "exact": 63,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D01",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 70,
   "exact": 65,
   "list_markers": 0,
   "spliced": 5,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D02",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 30,
   "exact": 30,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D02",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 29,
   "exact": 29,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D03",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 67,
   "exact": 67,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D03",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 68,
   "exact": 68,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D04",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 55,
   "exact": 55,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D04",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 67,
   "exact": 67,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D05",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 71,
   "exact": 71,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D05",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 45,
   "exact": 45,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D06",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 67,
   "exact": 67,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D06",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 68,
   "exact": 58,
   "list_markers": 0,
   "spliced": 10,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D07",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 59,
   "exact": 59,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D07",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 47,
   "exact": 47,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D08",
   "coder": "Sonnet",
   "parsed": true,
   "repaired": false,
   "quotes": 58,
   "exact": 58,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  },
  {
   "design": "D08",
   "coder": "Terra",
   "parsed": true,
   "repaired": false,
   "quotes": 61,
   "exact": 61,
   "list_markers": 0,
   "spliced": 0,
   "not_found": 0,
   "rule_issues": 0
  }
 ],
 "agreement_by_design": [
  {
   "design": "D01",
   "units": 9,
   "human_agree": 9,
   "machine_agree": 9,
   "output_agree": 9,
   "anyact_agree": 9,
   "acts_sonnet": 17,
   "acts_terra": 21,
   "acts_f1": 0.737,
   "acts_f1_merged": 0.737,
   "params_sonnet": 6,
   "params_terra": 7,
   "params_f1": 0.769,
   "branches_sonnet": 7,
   "branches_terra": 7,
   "branches_f1": 0.857,
   "loops_sonnet": 1,
   "loops_terra": 1
  },
  {
   "design": "D02",
   "units": 7,
   "human_agree": 7,
   "machine_agree": 7,
   "output_agree": 7,
   "anyact_agree": 7,
   "acts_sonnet": 5,
   "acts_terra": 5,
   "acts_f1": 1.0,
   "acts_f1_merged": 1.0,
   "params_sonnet": 1,
   "params_terra": 0,
   "params_f1": 0.0,
   "branches_sonnet": 1,
   "branches_terra": 1,
   "branches_f1": 1.0,
   "loops_sonnet": 0,
   "loops_terra": 0
  },
  {
   "design": "D03",
   "units": 6,
   "human_agree": 5,
   "machine_agree": 5,
   "output_agree": 5,
   "anyact_agree": 5,
   "acts_sonnet": 19,
   "acts_terra": 19,
   "acts_f1": 0.842,
   "acts_f1_merged": 0.895,
   "params_sonnet": 5,
   "params_terra": 5,
   "params_f1": 1.0,
   "branches_sonnet": 5,
   "branches_terra": 6,
   "branches_f1": 0.545,
   "loops_sonnet": 2,
   "loops_terra": 2
  },
  {
   "design": "D04",
   "units": 8,
   "human_agree": 8,
   "machine_agree": 7,
   "output_agree": 8,
   "anyact_agree": 8,
   "acts_sonnet": 11,
   "acts_terra": 17,
   "acts_f1": 0.643,
   "acts_f1_merged": 0.643,
   "params_sonnet": 5,
   "params_terra": 5,
   "params_f1": 0.6,
   "branches_sonnet": 3,
   "branches_terra": 1,
   "branches_f1": 0.5,
   "loops_sonnet": 1,
   "loops_terra": 1
  },
  {
   "design": "D05",
   "units": 7,
   "human_agree": 6,
   "machine_agree": 6,
   "output_agree": 5,
   "anyact_agree": 5,
   "acts_sonnet": 16,
   "acts_terra": 8,
   "acts_f1": 0.667,
   "acts_f1_merged": 0.667,
   "params_sonnet": 5,
   "params_terra": 5,
   "params_f1": 1.0,
   "branches_sonnet": 11,
   "branches_terra": 8,
   "branches_f1": 0.737,
   "loops_sonnet": 1,
   "loops_terra": 1
  },
  {
   "design": "D06",
   "units": 10,
   "human_agree": 7,
   "machine_agree": 10,
   "output_agree": 10,
   "anyact_agree": 7,
   "acts_sonnet": 21,
   "acts_terra": 19,
   "acts_f1": 0.65,
   "acts_f1_merged": 0.65,
   "params_sonnet": 11,
   "params_terra": 10,
   "params_f1": 0.857,
   "branches_sonnet": 2,
   "branches_terra": 5,
   "branches_f1": 0.571,
   "loops_sonnet": 1,
   "loops_terra": 1
  },
  {
   "design": "D07",
   "units": 8,
   "human_agree": 8,
   "machine_agree": 8,
   "output_agree": 5,
   "anyact_agree": 6,
   "acts_sonnet": 16,
   "acts_terra": 11,
   "acts_f1": 0.667,
   "acts_f1_merged": 0.667,
   "params_sonnet": 3,
   "params_terra": 3,
   "params_f1": 1.0,
   "branches_sonnet": 3,
   "branches_terra": 3,
   "branches_f1": 1.0,
   "loops_sonnet": 2,
   "loops_terra": 1
  },
  {
   "design": "D08",
   "units": 10,
   "human_agree": 10,
   "machine_agree": 10,
   "output_agree": 9,
   "anyact_agree": 9,
   "acts_sonnet": 12,
   "acts_terra": 10,
   "acts_f1": 0.636,
   "acts_f1_merged": 0.636,
   "params_sonnet": 3,
   "params_terra": 3,
   "params_f1": 1.0,
   "branches_sonnet": 9,
   "branches_terra": 13,
   "branches_f1": 0.636,
   "loops_sonnet": 1,
   "loops_terra": 1
  }
 ],
 "agreement_pooled": {
  "human": {
   "agree": 60,
   "of": 65,
   "share": 0.9230769230769231
  },
  "machine": {
   "agree": 62,
   "of": 65,
   "share": 0.9538461538461539
  },
  "output": {
   "agree": 58,
   "of": 65,
   "share": 0.8923076923076924
  },
  "anyact": {
   "agree": 56,
   "of": 65,
   "share": 0.8615384615384616
  },
  "units": 65,
  "acts": {
   "sonnet": 117,
   "terra": 110,
   "f1": 0.7136563876651982,
   "f1_merged": 0.7224669603524229
  },
  "params": {
   "sonnet": 39,
   "terra": 38,
   "f1": 0.8571428571428571
  },
  "branches": {
   "sonnet": 41,
   "terra": 44,
   "f1": 0.7058823529411765
  },
  "loops": {
   "sonnet": 9,
   "terra": 8
  }
 },
 "design_level_agreement": [
  {
   "measure": "Units with a human action (share)",
   "pearson": 0.8029487379470598,
   "spearman": 0.7297768432250877,
   "mean_sonnet": 0.8464285714285714,
   "mean_terra": 0.7702380952380953
  },
  {
   "measure": "Units with a machine action (share)",
   "pearson": 0.9063356658340498,
   "spearman": 0.7483357769119654,
   "mean_sonnet": 0.8601190476190477,
   "mean_terra": 0.8058035714285714
  },
  {
   "measure": "Units with a control act (share)",
   "pearson": 0.8609472521496901,
   "spearman": 0.8293893427471718,
   "mean_sonnet": 0.8339285714285715,
   "mean_terra": 0.6961309523809524
  },
  {
   "measure": "Units naming an output (share)",
   "pearson": 0.9493198991556333,
   "spearman": 0.9759000729485332,
   "mean_sonnet": 0.54375,
   "mean_terra": 0.42782738095238093
  },
  {
   "measure": "Judgement and decision acts per unit",
   "pearson": 0.7793837006516954,
   "spearman": 0.29091443373930537,
   "mean_sonnet": 1.0581845238095238,
   "mean_terra": 1.1074404761904761
  },
  {
   "measure": "Set acts per unit",
   "pearson": 0.6847681628469392,
   "spearman": 0.6468665190255972,
   "mean_sonnet": 0.37455357142857143,
   "mean_terra": 0.3115575396825397
  },
  {
   "measure": "Loops",
   "pearson": 0.8340576562282991,
   "spearman": 0.8189230248533257,
   "mean_sonnet": 1.125,
   "mean_terra": 1
  },
  {
   "measure": "Named tunable parameters",
   "pearson": 0.9765193752894448,
   "spearman": 1.0,
   "mean_sonnet": 4.875,
   "mean_terra": 4.75
  },
  {
   "measure": "Branches",
   "pearson": 0.8168809919154582,
   "spearman": 0.8734939759036144,
   "mean_sonnet": 5.125,
   "mean_terra": 5.5
  }
 ],
 "tables": {
  "t1": [
   [
    "Loops stated",
    "0.75 / 0.75",
    "1.50 / 1.25",
    "+0.75 / +0.50",
    "3/0 ; 2/0",
    "consistent increase"
   ],
   [
    "Annotation / logging acts",
    "1.50 / 0.75",
    "3.00 / 2.50",
    "+1.50 / +1.75",
    "3/0 ; 4/0",
    "consistent increase"
   ],
   [
    "Provenance statements",
    "0.00 / 0.00",
    "0.50 / 0.75",
    "+0.50 / +0.75",
    "2/0 ; 3/0",
    "increase (none in C1)"
   ],
   [
    "Units naming an output (share)",
    "0.37 / 0.35",
    "0.71 / 0.51",
    "+0.34 / +0.16",
    "3/1 ; 2/1",
    "increase"
   ],
   [
    "Branches stated",
    "3.75 / 5.00",
    "6.50 / 6.00",
    "+2.75 / +1.00",
    "2/1 ; 3/1",
    "higher, but mixed by block"
   ],
   [
    "Judgement / decision acts",
    "6.75 / 9.50",
    "9.50 / 7.75",
    "+2.75 / -1.75",
    "3/1 ; 1/3",
    "unstable (coders differ in sign)"
   ],
   [
    "Named tunable parameters",
    "5.00 / 4.50",
    "4.75 / 5.00",
    "-0.25 / +0.50",
    "2/2 ; 2/2",
    "no clear effect"
   ],
   [
    "Units with a control act (share)",
    "0.83 / 0.73",
    "0.84 / 0.66",
    "+0.01 / -0.06",
    "2/1 ; 2/2",
    "no clear effect"
   ],
   [
    "Units with human action (share)",
    "0.85 / 0.78",
    "0.84 / 0.76",
    "-0.01 / -0.02",
    "2/1 ; 2/1",
    "no clear effect"
   ],
   [
    "Units with machine action (share)",
    "0.80 / 0.77",
    "0.92 / 0.84",
    "+0.11 / +0.07",
    "2/1 ; 3/1",
    "small increase, mixed"
   ],
   [
    "Judgements with criterion stated (share)",
    "0.44 / 0.48",
    "0.40 / 0.53",
    "-0.04 / +0.05",
    "2/2 ; 2/2",
    "no clear effect"
   ],
   [
    "Loops stating an exit",
    "0.00 / 0.00",
    "0.00 / 0.00",
    "+0.00 / +0.00",
    "0/0 ; 0/0",
    "none in either condition"
   ]
  ],
  "t2": [
   [
    "Units per design",
    "7.50 / 7.50",
    "8.75 / 8.75",
    "7.50 / 7.50",
    "8.75 / 8.75"
   ],
   [
    "Units with a control act (share)",
    "0.77 / 0.63",
    "0.90 / 0.76",
    "0.79 / 0.65",
    "0.88 / 0.74"
   ],
   [
    "Units with machine action (share)",
    "0.93 / 0.86",
    "0.79 / 0.75",
    "0.83 / 0.75",
    "0.89 / 0.86"
   ],
   [
    "Named tunable parameters",
    "3.50 / 3.25",
    "6.25 / 6.25",
    "3.50 / 3.25",
    "6.25 / 6.25"
   ],
   [
    "Branches stated",
    "4.50 / 3.25",
    "5.75 / 7.75",
    "6.50 / 7.00",
    "3.75 / 4.00"
   ],
   [
    "Loops stated",
    "1.00 / 0.75",
    "1.25 / 1.25",
    "1.00 / 1.00",
    "1.25 / 1.00"
   ]
  ],
  "t3": [
   [
    "Transforms",
    "18.8",
    "24.2",
    "35.2",
    "43.8",
    "31.8"
   ],
   [
    "Human share",
    "0.28",
    "0.42",
    "0.30",
    "0.35",
    "0.28"
   ],
   [
    "Machine share",
    "0.59",
    "0.54",
    "0.60",
    "0.58",
    "0.54"
   ],
   [
    "Hybrid share",
    "0.13",
    "0.05",
    "0.09",
    "0.07",
    "0.17"
   ],
   [
    "Loops",
    "1.75",
    "2.75",
    "2.25",
    "4.25",
    "2.75"
   ],
   [
    "Loops with an until",
    "1.75",
    "2.75",
    "2.25",
    "4.25",
    "2.75"
   ],
   [
    "Specification artifacts",
    "3.75",
    "6.25",
    "6.00",
    "14.25",
    "5.25"
   ],
   [
    "Specifications revised in a loop",
    "1.50",
    "3.00",
    "2.25",
    "5.00",
    "2.25"
   ],
   [
    "Conditional blocks outside loops",
    "0.00",
    "0.00",
    "0.00",
    "0.25",
    "0.25"
   ],
   [
    "Human/hybrid judgement transforms",
    "3.75",
    "5.50",
    "6.50",
    "8.50",
    "6.00"
   ],
   [
    "Human/hybrid decision transforms",
    "2.00",
    "4.25",
    "4.50",
    "6.75",
    "4.50"
   ]
  ],
  "t4": [
   [
    "Loops per design",
    "0.75",
    "1.75",
    "1.38",
    "2.75",
    "+1.00 (3/0)",
    "+0.62 (3/0)",
    "+1.00 (3/0)",
    "+1.38 (3/0)",
    "+0.38 (2/1)"
   ],
   [
    "Human judgement / decision points",
    "8.12",
    "5.75",
    "8.62",
    "9.75",
    "-2.38 (1/3)",
    "+0.50 (3/1)",
    "+4.00 (4/0)",
    "+1.12 (2/2)",
    "+3.50 (3/1)"
   ],
   [
    "Tunable parameters (prose) / specifications revised in loops (ATWL)",
    "3.38",
    "1.50",
    "3.38",
    "3.00",
    "-1.88 (2/2)",
    "+0.00 (2/1)",
    "+1.50 (3/0)",
    "-0.38 (2/2)",
    "+1.50 (2/2)"
   ],
   [
    "Conditional branches (prose) / conditional blocks outside loops (ATWL)",
    "4.38",
    "0.00",
    "6.25",
    "0.00",
    "-4.38 (0/4)",
    "+1.88 (3/1)",
    "+0.00 (0/0)",
    "-6.25 (0/4)",
    "-1.88 (1/3)"
   ]
  ],
  "t5": [
   [
    "Loops",
    "VAST / Sonnet",
    "1.0",
    "2",
    "5",
    "3"
   ],
   [
    "Loops",
    "VAST / Terra",
    "2.0",
    "2",
    "3",
    "2"
   ],
   [
    "Loops",
    "VIS / Sonnet",
    "1.5",
    "2",
    "5",
    "3"
   ],
   [
    "Loops",
    "VIS / Terra",
    "1.0",
    "3",
    "4",
    "3"
   ],
   [
    "Human judgement / decision points",
    "VAST / Sonnet",
    "5.5",
    "12",
    "18",
    "20"
   ],
   [
    "Human judgement / decision points",
    "VAST / Terra",
    "15.0",
    "11",
    "14",
    "6"
   ],
   [
    "Human judgement / decision points",
    "VIS / Sonnet",
    "5.0",
    "12",
    "18",
    "9"
   ],
   [
    "Human judgement / decision points",
    "VIS / Terra",
    "9.0",
    "9",
    "11",
    "7"
   ],
   [
    "Conditional paths",
    "VAST / Sonnet",
    "7.0",
    "0",
    "0",
    "1"
   ],
   [
    "Conditional paths",
    "VAST / Terra",
    "3.0",
    "0",
    "1",
    "0"
   ],
   [
    "Conditional paths",
    "VIS / Sonnet",
    "1.0",
    "0",
    "0",
    "0"
   ],
   [
    "Conditional paths",
    "VIS / Terra",
    "5.5",
    "0",
    "0",
    "0"
   ],
   [
    "Tunable parameters / specs revised in loops",
    "VAST / Sonnet",
    "3.5",
    "2",
    "5",
    "3"
   ],
   [
    "Tunable parameters / specs revised in loops",
    "VAST / Terra",
    "1.5",
    "2",
    "4",
    "2"
   ],
   [
    "Tunable parameters / specs revised in loops",
    "VIS / Sonnet",
    "2.5",
    "2",
    "4",
    "2"
   ],
   [
    "Tunable parameters / specs revised in loops",
    "VIS / Terra",
    "6.0",
    "3",
    "7",
    "2"
   ]
  ],
  "t6": [
   [
    "Units with a stated actor",
    "0.88",
    "1.00",
    "1.00 (declared by syntax)"
   ],
   [
    "Units naming their output",
    "0.36",
    "0.61",
    "1.00 (declared by syntax)"
   ],
   [
    "Loops stating an exit condition",
    "0 of 6",
    "0 of 11",
    "55 of 55 declare an until"
   ],
   [
    "Loops stating what triggers another round",
    "3 of 6",
    "4 of 11",
    "not coded in ATWL"
   ],
   [
    "Parameters with an initial value stated",
    "11 of 38",
    "9 of 39",
    "not comparable"
   ],
   [
    "Judgements with a criterion stated",
    "0.46",
    "0.47",
    "no field in ATWL"
   ]
  ]
 },
 "robustness": {
  "loops": {
   "min": {
    "mean_diff": 0.5,
    "up": 2,
    "down": 0
   },
   "max": {
    "mean_diff": 0.75,
    "up": 3,
    "down": 0
   }
  },
  "annotate": {
   "min": {
    "mean_diff": 1.25,
    "up": 4,
    "down": 0
   },
   "max": {
    "mean_diff": 2,
    "up": 3,
    "down": 0
   }
  },
  "prov": {
   "min": {
    "mean_diff": 0.5,
    "up": 2,
    "down": 0
   },
   "max": {
    "mean_diff": 0.75,
    "up": 3,
    "down": 0
   }
  },
  "branches": {
   "min": {
    "mean_diff": 2.5,
    "up": 3,
    "down": 1
   },
   "max": {
    "mean_diff": 1.25,
    "up": 2,
    "down": 1
   }
  },
  "params": {
   "min": {
    "mean_diff": 0.25,
    "up": 2,
    "down": 2
   },
   "max": {
    "mean_diff": 0,
    "up": 2,
    "down": 2
   }
  },
  "output_named_share": {
   "min": {
    "mean_diff": 0.15922619047619047,
    "up": 2,
    "down": 1
   },
   "max": {
    "mean_diff": 0.3410714285714285,
    "up": 3,
    "down": 1
   }
  },
  "machine_share": {
   "min": {
    "mean_diff": 0.0669642857142857,
    "up": 3,
    "down": 1
   },
   "max": {
    "mean_diff": 0.11309523809523808,
    "up": 2,
    "down": 1
   }
  },
  "judge_decide": {
   "min": {
    "mean_diff": 0.5,
    "up": 2,
    "down": 2
   },
   "max": {
    "mean_diff": 0.5,
    "up": 2,
    "down": 2
   }
  }
 },
 "atwl_loops_total": 55,
 "atwl_loops_until": 55
}