Effect size computation: pinillos2012knowledgeexperimentspracitcal

library(esc)
library(metafor)

Loading required package: Matrix

Loading required package: metadat

Loading required package: numDeriv

Loading the 'metafor' package (version 4.8-0). For an
introduction to the package please type: help(metafor)

Shared helpers

stop_if_missing <- function(x, name) {
  if (is.na(x)) stop(sprintf("Missing required input: %s", name), call. = FALSE)
}

# Exact small-sample correction factor used by metafor (.cmicalc).
hedges_correction <- function(df) {
  ifelse(df <= 1, NA_real_, exp(lgamma(df / 2) - log(sqrt(df / 2)) - lgamma((df - 1) / 2)))
}

compute_effect_size <- function(
    paper_key,
    study_id,
    effect_id,
    method_used,
    sign_convention = "d = mean(low) - mean(high)",
    n_high = NA_real_,
    n_low = NA_real_,
    n_total = NA_real_,
    t_value = NA_real_,
    reported_r = NA_real_,
    sign_d = NA_real_,
    notes_on_assumptions = "",
    imputed_flag = FALSE,
    needs_sensitivity = TRUE
) {
  d <- NA_real_
  v <- NA_real_
  g <- NA_real_
  v_g <- NA_real_
  computed_from_suggested <- NA_character_
  design_used <- if (startsWith(method_used, "between_")) {
    "Between-Subjects"
  } else if (startsWith(method_used, "within_")) {
    "Within-Subjects"
  } else {
    NA_character_
  }

  if (method_used == "between_rpb") {
    computed_from_suggested <- "other"
    stop_if_missing(reported_r, "reported_r")
    stop_if_missing(sign_d, "sign_d")
    if (!is.element(sign_d, c(-1, 1))) stop("sign_d must be +1 or -1", call. = FALSE)

    # Preferred: use split group Ns when known.
    if (!is.na(n_high) && !is.na(n_low)) {
      d_out <- esc::esc_rpb(r = abs(reported_r), grp1n = n_low, grp2n = n_high, es.type = "d")
      g_out <- esc::esc_rpb(r = abs(reported_r), grp1n = n_low, grp2n = n_high, es.type = "g")
    } else {
      # Fallback requested for this paper: if only total N is available,
      # assume equal split (n_low = n_high = totaln / 2) via esc_rpb(totaln=...).
      stop_if_missing(n_total, "n_total")
      d_out <- esc::esc_rpb(r = abs(reported_r), totaln = n_total, es.type = "d")
      g_out <- esc::esc_rpb(r = abs(reported_r), totaln = n_total, es.type = "g")
      if (is.na(n_low)) n_low <- n_total / 2
      if (is.na(n_high)) n_high <- n_total / 2
    }

    d <- sign_d * abs(d_out$es)
    v <- d_out$var
    g <- sign_d * abs(g_out$es)
    v_g <- g_out$var
  } else if (method_used == "within_smcc_t") {
    computed_from_suggested <- "t_df"
    stop_if_missing(n_total, "n_total")
    stop_if_missing(t_value, "t_value")

    esc_out <- metafor::escalc(measure = "SMCC", ti = t_value, ni = n_total)
    d <- esc_out$yi
    v <- esc_out$vi

    df_used <- n_total - 1
    J <- hedges_correction(df_used)
    g <- J * d
    v_g <- (J^2) * v
  } else {
    stop(sprintf("Unknown method_used: %s", method_used), call. = FALSE)
  }

  inputs_used <- paste(
    c(
      sprintf("method=%s", method_used),
      sprintf("sign_convention=%s", sign_convention),
      if (!is.na(n_low)) sprintf("n_low=%s", n_low) else NULL,
      if (!is.na(n_high)) sprintf("n_high=%s", n_high) else NULL,
      if (!is.na(n_total)) sprintf("n_total=%s", n_total) else NULL,
      if (!is.na(t_value)) sprintf("t=%s", t_value) else NULL,
      if (!is.na(reported_r)) sprintf("reported_r=%s", reported_r) else NULL,
      if (!is.na(sign_d)) sprintf("sign_d=%s", sign_d) else NULL
    ),
    collapse = ", "
  )

  audit <- data.frame(
    paper_key = paper_key,
    study_id = study_id,
    effect_id = effect_id,
    design = design_used,
    method_used = method_used,
    computed_from_suggested = computed_from_suggested,
    inputs_used = inputs_used,
    d = d,
    v = v,
    g = g,
    v_g = v_g,
    notes_on_assumptions = notes_on_assumptions,
    imputed_flag = imputed_flag,
    needs_sensitivity = needs_sensitivity
  )

  yaml_snippet <- sprintf(
    "effect_size:\\n  metric: SMD\\n  d: %.12f\\n  v: %.12f\\n  computed_from: %s\\n  needs_review: false\\n  notes: \\\"%s\\\"\\n",
    d, v, computed_from_suggested, gsub(pattern = "\"", replacement = "'", x = inputs_used)
  )

  list(audit = audit, yaml_snippet = yaml_snippet)
}

Study 1: Typo (Between-Subjects)

Effect s1_e1: Typo-Low vs Typo-High (Mann-Whitney r -> d)

paper_key <- "pinillos2012knowledgeexperimentspracitcal"
study_id <- 1
effect_id <- "s1_e1"

inputs <- list(
  n_low = 77,
  n_high = 67,
  reported_r = -0.56,
  sign_d = -1
)
inputs

$n_low
[1] 77

$n_high
[1] 67

$reported_r
[1] -0.56

$sign_d
[1] -1

res_s1_e1 <- compute_effect_size(
  paper_key = paper_key,
  study_id = study_id,
  effect_id = effect_id,
  method_used = "between_rpb",
  n_high = inputs$n_high,
  n_low = inputs$n_low,
  reported_r = inputs$reported_r,
  sign_d = inputs$sign_d,
  notes_on_assumptions = "Computed with esc::esc_rpb from reported Mann-Whitney r using known split Ns."
)
res_s1_e1$audit

                                  paper_key study_id effect_id           design
1 pinillos2012knowledgeexperimentspracitcal        1     s1_e1 Between-Subjects
  method_used computed_from_suggested
1 between_rpb                   other
                                                                                                       inputs_used
1 method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=77, n_high=67, reported_r=-0.56, sign_d=-1
          d          v         g        v_g
1 -1.355124 0.03428864 -1.347954 0.03428864
                                                           notes_on_assumptions
1 Computed with esc::esc_rpb from reported Mann-Whitney r using known split Ns.
  imputed_flag needs_sensitivity
1        FALSE              TRUE

cat(res_s1_e1$yaml_snippet)

effect_size:\n  metric: SMD\n  d: -1.355124252114\n  v: 0.034288642158\n  computed_from: other\n  needs_review: false\n  notes: \"method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=77, n_high=67, reported_r=-0.56, sign_d=-1\"\n

Effect s1_e2: Typo-Low vs Ignorant-Typo-High (Mann-Whitney r -> d)

paper_key <- "pinillos2012knowledgeexperimentspracitcal"
study_id <- 1
effect_id <- "s1_e2"

inputs <- list(
  n_low = 77,
  n_high = 69,
  reported_r = -0.39,
  sign_d = -1
)
inputs

$n_low
[1] 77

$n_high
[1] 69

$reported_r
[1] -0.39

$sign_d
[1] -1

res_s1_e2 <- compute_effect_size(
  paper_key = paper_key,
  study_id = study_id,
  effect_id = effect_id,
  method_used = "between_rpb",
  n_high = inputs$n_high,
  n_low = inputs$n_low,
  reported_r = inputs$reported_r,
  sign_d = inputs$sign_d,
  notes_on_assumptions = "Computed with esc::esc_rpb from reported Mann-Whitney r using known split Ns."
)
res_s1_e2$audit

                                  paper_key study_id effect_id           design
1 pinillos2012knowledgeexperimentspracitcal        1     s1_e2 Between-Subjects
  method_used computed_from_suggested
1 between_rpb                   other
                                                                                                       inputs_used
1 method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=77, n_high=69, reported_r=-0.39, sign_d=-1
           d          v          g        v_g
1 -0.8483503 0.02994449 -0.8439242 0.02994449
                                                           notes_on_assumptions
1 Computed with esc::esc_rpb from reported Mann-Whitney r using known split Ns.
  imputed_flag needs_sensitivity
1        FALSE              TRUE

cat(res_s1_e2$yaml_snippet)

effect_size:\n  metric: SMD\n  d: -0.848350338028\n  v: 0.029944486802\n  computed_from: other\n  needs_review: false\n  notes: \"method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=77, n_high=69, reported_r=-0.39, sign_d=-1\"\n

Study 1 follow-up: Belief fixed (Between-Subjects)

Effect s2_e1: Belief-Typo-Low vs Belief-Typo-High (Mann-Whitney r -> d)

paper_key <- "pinillos2012knowledgeexperimentspracitcal"
study_id <- 2
effect_id <- "s2_e1"

inputs <- list(
  n_total = 78,
  reported_r = 0.34,
  sign_d = -1
)
inputs

$n_total
[1] 78

$reported_r
[1] 0.34

$sign_d
[1] -1

res_s2_e1 <- compute_effect_size(
  paper_key = paper_key,
  study_id = study_id,
  effect_id = effect_id,
  method_used = "between_rpb",
  n_total = inputs$n_total,
  reported_r = inputs$reported_r,
  sign_d = inputs$sign_d,
  notes_on_assumptions = "Computed with esc::esc_rpb from reported Mann-Whitney r. Split group Ns were not reported; equal split (n_low=n_high=totaln/2) assumed."
)
res_s2_e1$audit

                                  paper_key study_id effect_id           design
1 pinillos2012knowledgeexperimentspracitcal        2     s2_e1 Between-Subjects
  method_used computed_from_suggested
1 between_rpb                   other
                                                                                                                  inputs_used
1 method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=39, n_high=39, n_total=78, reported_r=0.34, sign_d=-1
          d          v          g        v_g
1 -0.723077 0.05463359 -0.7159178 0.05463359
                                                                                                                     notes_on_assumptions
1 Computed with esc::esc_rpb from reported Mann-Whitney r. Split group Ns were not reported; equal split (n_low=n_high=totaln/2) assumed.
  imputed_flag needs_sensitivity
1        FALSE              TRUE

cat(res_s2_e1$yaml_snippet)

effect_size:\n  metric: SMD\n  d: -0.723076997100\n  v: 0.054633591947\n  computed_from: other\n  needs_review: false\n  notes: \"method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=39, n_high=39, n_total=78, reported_r=0.34, sign_d=-1\"\n

Study 2: Typo juxtaposed (Within-Subjects)

Effect s3_e1: Juxtaposed Typo (paired t -> SMCC)

paper_key <- "pinillos2012knowledgeexperimentspracitcal"
study_id <- 3
effect_id <- "s3_e1"

inputs <- list(
  n_total = 95,
  t_value = -7.6
)
inputs

$n_total
[1] 95

$t_value
[1] -7.6

res_s3_e1 <- compute_effect_size(
  paper_key = paper_key,
  study_id = study_id,
  effect_id = effect_id,
  method_used = "within_smcc_t",
  n_total = inputs$n_total,
  t_value = inputs$t_value,
  notes_on_assumptions = "Use metafor::escalc(measure='SMCC') with paired t and n_total; within-subject fallback (no matching esc method for this project metric)."
)
res_s3_e1$audit

                                  paper_key study_id effect_id          design
1 pinillos2012knowledgeexperimentspracitcal        3     s3_e1 Within-Subjects
    method_used computed_from_suggested
1 within_smcc_t                    t_df
                                                                           inputs_used
1 method=within_smcc_t, sign_convention=d = mean(low) - mean(high), n_total=95, t=-7.6
           d         v         g        v_g
1 -0.7735028 0.0136753 -0.767312 0.01345727
                                                                                                                      notes_on_assumptions
1 Use metafor::escalc(measure='SMCC') with paired t and n_total; within-subject fallback (no matching esc method for this project metric).
  imputed_flag needs_sensitivity
1        FALSE              TRUE

cat(res_s3_e1$yaml_snippet)

effect_size:\n  metric: SMD\n  d: -0.773502819918\n  v: 0.013675297960\n  computed_from: t_df\n  needs_review: false\n  notes: \"method=within_smcc_t, sign_convention=d = mean(low) - mean(high), n_total=95, t=-7.6\"\n

Study 3: Counting contest knowledge prompt (Between-Subjects)

Effect s4_e1: Count-Low-Knowledge vs Count-High-Knowledge (Mann-Whitney r -> d)

paper_key <- "pinillos2012knowledgeexperimentspracitcal"
study_id <- 4
effect_id <- "s4_e1"

inputs <- list(
  n_total = 128,
  reported_r = -0.40,
  sign_d = -1
)
inputs

$n_total
[1] 128

$reported_r
[1] -0.4

$sign_d
[1] -1

res_s4_e1 <- compute_effect_size(
  paper_key = paper_key,
  study_id = study_id,
  effect_id = effect_id,
  method_used = "between_rpb",
  n_total = inputs$n_total,
  reported_r = inputs$reported_r,
  sign_d = inputs$sign_d,
  notes_on_assumptions = "Computed with esc::esc_rpb from reported Mann-Whitney r. Split group Ns were not reported; equal split (n_low=n_high=totaln/2) assumed."
)
res_s4_e1$audit

                                  paper_key study_id effect_id           design
1 pinillos2012knowledgeexperimentspracitcal        4     s4_e1 Between-Subjects
  method_used computed_from_suggested
1 between_rpb                   other
                                                                                                                   inputs_used
1 method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=64, n_high=64, n_total=128, reported_r=-0.4, sign_d=-1
           d          v          g        v_g
1 -0.8728716 0.03422619 -0.8676656 0.03422619
                                                                                                                     notes_on_assumptions
1 Computed with esc::esc_rpb from reported Mann-Whitney r. Split group Ns were not reported; equal split (n_low=n_high=totaln/2) assumed.
  imputed_flag needs_sensitivity
1        FALSE              TRUE

cat(res_s4_e1$yaml_snippet)

effect_size:\n  metric: SMD\n  d: -0.872871560944\n  v: 0.034226190476\n  computed_from: other\n  needs_review: false\n  notes: \"method=between_rpb, sign_convention=d = mean(low) - mean(high), n_low=64, n_high=64, n_total=128, reported_r=-0.4, sign_d=-1\"\n