I'm not sure but here is the best solution I can provide:

I feel sort of optimization should be used to solve this issue, or definitely better `model` (rather then linear OLS model) but nonlinear...

    diff_binAB <- with(data, unique(diff_AB))
    mse <- numeric(length(diff_binAB))

    for(i in 1:length(diff_binAB)){
        pwise <- with(data, lm(retA ~ diff_AB*(diff_AB < diff_binAB[i]) + diff_AB*(diff_AB >= diff_binAB[i])))
        mse[i] <- summary(pwise)[6]
       }

    mse <- as.numeric(mse)
    mse 

    diff_binAB[which(mse==min(mse))]
    # -0.07

    diff_binAC <- with(data, unique(diff_AC))
    mse1 <- numeric(length(diff_binAC))

    for(i in 1:length(diff_binAC)){
  
        pwise <- with(data, lm(retA ~ diff_AC*(diff_AC < diff_binAC[i]) + diff_AC*(diff_AB >= diff_binAC[i])))
        mse1[i] <- summary(pwise)[6]
       }

    mse1 <- as.numeric(mse1)
    mse1 

    diff_binAC[which(mse1==min(mse1))]
    # 0.04 

Here the results would suggest that the `return` (rate of change) is `explained` if the difference between `A` and `B` is at `-0.07` (negative difference) and `0.04` with possitive difference between `A` and `C`.