On Mon, Sep 24, 2012 at 08:35:15PM +0200, Steinar H. Gunderson wrote:
> I could try to hack it so that MODE_LUMA becomes identical to MODE_RGB?

I means s/MODE/CHANNEL/g, of course. But here's a patch that obsoletes
CHANNEL_LUMA.

/* Steinar */
--
Homepage: http://www.sesse.net/

>From 9cae93803a0b7f256d8d45435dd15798371d5f26 Mon Sep 17 00:00:00 2001
From: "Steinar H. Gunderson" <sgunderson@???>
Date: Mon, 24 Sep 2012 20:38:33 +0200
Subject: [PATCH] Remove the luma option from the curves filter.

Like in the levels plugin, the luma option (CHANNEL_LUMA) is broken
by design. Fortunately, there is already an option that does the
right thing (CHANNEL_RGB), so if CHANNEL_LUMA is given, treat it
as CHANNEL_RGB everywhere.
---
src/filter/curves/curves.c | 50 ++++++++------------------------------------
1 files changed, 9 insertions(+), 41 deletions(-)

diff --git a/src/filter/curves/curves.c b/src/filter/curves/curves.c
index f2fa96f..87b5707 100644
--- a/src/filter/curves/curves.c
+++ b/src/filter/curves/curves.c
@@ -206,7 +206,7 @@ void f0r_get_param_info(f0r_param_info_t* info, int param_index)
   case 0:
     info->name = "Channel";
     info->type = F0R_PARAM_DOUBLE;
-    info->explanation = "Channel to adjust (0 = red, 0.1 = green, 0.2 = blue, 0.3 = alpha, 0.4 = luma, 0.5 = rgb, 0.6 = hue, 0.7 = saturation)";
+    info->explanation = "Channel to adjust (0 = red, 0.1 = green, 0.2 = blue, 0.3 = alpha, 0.4 = <obsolete>, 0.5 = rgb, 0.6 = hue, 0.7 = saturation)";
     break;
   case 1:
     info->name = "Show curves";
@@ -226,7 +226,7 @@ void f0r_get_param_info(f0r_param_info_t* info, int param_index)
   case 4:
     info->name = "Luma formula";
     info->type = F0R_PARAM_BOOL;
-    info->explanation = "Use Rec. 601 (false) or Rec. 709 (true)";
+    info->explanation = "Obsolete; has no effect.";
     break;
   case 5:
     info->name = "Bézier spline";
@@ -290,8 +290,8 @@ void f0r_set_param_value(f0r_instance_t instance,
           if (tmp >= 1) {
               // legacy support
               if (tmp == 3) {
-                  if (inst->channel != CHANNEL_LUMA) {
-                    inst->channel = CHANNEL_LUMA;
+                  if (inst->channel != CHANNEL_RGB) {
+                    inst->channel = CHANNEL_RGB;
                     if (strlen(inst->bspline))
                         updateBsplineMap(instance);
                   }
@@ -578,9 +578,9 @@ void updateBsplineMap(f0r_instance_t instance)
     if (inst->channel == CHANNEL_HUE) {
         for(int i = 0; i < 361; ++i)
             inst->bsplineMap[i] = i;
-    } else if (inst->channel == CHANNEL_LUMA || inst->channel == CHANNEL_SATURATION) {
+    } else if (inst->channel == CHANNEL_SATURATION) {
         for(int i = 0; i < 256; ++i)
-            inst->bsplineMap[i] = inst->channel == CHANNEL_LUMA ? 1 : i / 255.;
+            inst->bsplineMap[i] = i / 255.;
     } else {
         for(int i = 0; i < 256; ++i)
             inst->bsplineMap[i] = i;
@@ -674,8 +674,6 @@ void updateBsplineMap(f0r_instance_t instance)

             if (inst->channel == CHANNEL_HUE)
                 inst->bsplineMap[j] = CLAMP(y * 360, 0, 360);
-            else if (inst->channel == CHANNEL_LUMA)
-                inst->bsplineMap[j] = y / (j == 0 ? 1 : j / 255.);
             else if (inst->channel == CHANNEL_SATURATION)
                 inst->bsplineMap[j] = CLAMP(y, 0, 1);
             else
@@ -743,11 +741,8 @@ void f0r_update(f0r_instance_t instance, double time,
       map = malloc(361*sizeof(double));
       memcpy(map, inst->bsplineMap, (inst->channel == CHANNEL_HUE ? 361 : 256)*sizeof(double));
       if (inst->channel != CHANNEL_SATURATION && inst->channel != CHANNEL_HUE) {
-          if (inst->channel == CHANNEL_LUMA)
-              memcpy(mapLuma, map, 256*sizeof(double));
-          else
-              for (i = 0; i < 256; ++i)
-                  mapI[i] = (int)map[i];
+          for (i = 0; i < 256; ++i)
+              mapI[i] = (int)map[i];
       }
   }

@@ -756,6 +751,7 @@ void f0r_update(f0r_instance_t instance, double time,
double rf, gf, bf, hue, sat, val;

   switch ((int)inst->channel) {
+  case CHANNEL_LUMA:
   case CHANNEL_RGB:
       while (len--) {
           *dst++ = mapI[*src++];        // r
@@ -795,34 +791,6 @@ void f0r_update(f0r_instance_t instance, double time,
           dst += 4;
       }
       break;
-  case CHANNEL_LUMA:
-      if (inst->formula) {      // Rec.709
-          factorR = .2126;
-          factorG = .7152;
-          factorB = .0722;
-      } else {                  // Rec. 601
-          factorR = .299;
-          factorG = .587;
-          factorB = .114;
-      }
-      while (len--) {
-          r = *src++;
-          g = *src++;
-          b = *src++;
-          luma = ROUND(factorR * r + factorG * g + factorB * b);
-          lumaValue = mapLuma[luma];
-          if (luma == 0) {
-              *dst++ = lumaValue;
-              *dst++ = lumaValue;
-              *dst++ = lumaValue;
-          } else {
-              *dst++ = CLAMP0255((int)(r * lumaValue));
-              *dst++ = CLAMP0255((int)(g * lumaValue));
-              *dst++ = CLAMP0255((int)(b * lumaValue));
-          }
-          *dst++ = *src++;
-      }
-      break;
   case CHANNEL_HUE:
       while (len--) {
           rf = *src++;
-- 
1.7.7.3