how to write a cga shader in glsl?

Question

How to write a CGA shader that limit the palette to 4 colors and match the original colors with those (cyan, magenta, black, white)??

I'm working on Game Maker Studio Professional, that actually allows the using of shaders writing vertex and fragment code

I already asked this question also in the yoyogames community and in the openGl forum

someone answered me with this :

varying vec2 v_vTexcoord;
varying vec4 v_vColour;

const mat3 rgb_to_wcm = mat3(1,-1, 0,  1, 0,-1, -1, 1, 1);

void main()
{
    vec4 rgba = v_vColour * texture2D(gm_BaseTexture, v_vTexcoord);
    vec3 wcm = rgb_to_wcm * rgba.rgb;
    vec3 rgb = dot(wcm,vec3(1,1,1)) < 0.5
        ? vec3(0,0,0)
        : wcm.x > wcm.y
            ? (wcm.x > wcm.z ? vec3(1,1,1) : vec3(1,0,1))
            : (wcm.y > wcm.z ? vec3(0,1,1) : vec3(1,0,1));
    gl_FragColor = vec4(rgb, rgba.a);
}

it works, but not return a cga palette, it return a black and white (not grayscale)

how can I do?

Answer 1

Thanks a lot I did in a different way, maybe more simple

https://www.shadertoy.com/view/MdlyDH

//   ▄████████    ▄██████▄     ▄████████ 
//  ███    ███   ███    ███   ███    ███ 
//  ███    █▀    ███    █▀    ███    ███ 
//  ███         ▄███          ███    ███ 
//  ███        ▀▀███ ████▄  ▀███████████ 
//  ███    █▄    ███    ███   ███    ███ 
//  ███    ███   ███    ███   ███    ███ 
//  ████████▀    ████████▀    ███    █▀  
// 

/*

////only for game maker studio////

varying vec2 v_vTexcoord;
varying vec4 v_vColour;

varying vec2 fragCoord;
//you have to put varying vec2 fragCoord also in the vertex shader
//after write in the last row of the local scope of the vertex shader: fragCoord = in_Position.xy

uniform vec2 iResolution;
uniform float iGlobalTime;

//palette 0 is not cga but gameboy, I put it as bonus
uniform int palette;
uniform float gamma;

*/

// rgb to float
// rgb to float = rgb / 255

// 0 = 0.0
// 85 = 0.333
// 170 = 0.666
// 255 = 1.0

void mainImage( out vec4 fragColor, in vec2 fragCoord )
{
    vec2 uv = fragCoord.xy / iResolution.xy;

    vec3 c = vec3(0.0);
    float alpha = 1.0;

    //only for shadertoy
    int palette = 2; //the number between 0 and 6 change palette
    float gamma = 1.5; //the gamma change the threshold of the palette swapper

    //c = texture2D(gm_BaseTexture,uv).rgb;
    c = texture(iChannel0,uv).rgb;

    c.r = pow(abs(c.r),gamma);
    c.g = pow(abs(c.g),gamma);
    c.b = pow(abs(c.b),gamma);

    vec3 col1 = vec3(0.0);
    vec3 col2 = vec3(0.0);
    vec3 col3 = vec3(0.0);
    vec3 col4 = vec3(0.0);

    if(palette == 0) {
        col1 = vec3(0.612,0.725,0.086);
        col2 = vec3(0.549,0.667,0.078);
        col3 = vec3(0.188,0.392,0.188);
        col4 = vec3(0.063,0.247,0.063);
    }
    if(palette == 1) {
        col1 = vec3(0.0);
        col2 = vec3(0.0,0.666,0.666);
        col3 = vec3(0.666,0.0,0.666);
        col4 = vec3(0.666,0.666,0.666);
    }
    if(palette == 2) {
        col1 = vec3(0.0);
        col2 = vec3(0.333,1.0,1.0);
        col3 = vec3(1.0,0.333,1.0);
        col4 = vec3(1.0);
    }
    if(palette == 3) {
        col1 = vec3(0.0);
        col2 = vec3(0.0,0.666,0.0);
        col3 = vec3(0.666,0.0,0.0);
        col4 = vec3(0.666,0.333,0.0);
    }
    if(palette == 4) {
        col1 = vec3(0.0);
        col2 = vec3(0.333,1.0,0.333);
        col3 = vec3(1.0,0.333,0.333);
        col4 = vec3(1.0,1.0,0.333);
    }
    if(palette == 5) {
        col1 = vec3(0.0);
        col2 = vec3(0.0,0.666,0.666);
        col3 = vec3(0.666,0.0,0.0);
        col4 = vec3(0.666,0.666,0.666);
    }
    if(palette == 6) {
        col1 = vec3(0.0);
        col2 = vec3(0.333,0.666,0.666);
        col3 = vec3(1.0,0.333,0.333);
        col4 = vec3(1.0);
    }

    float dist1 = length(c - col1);
    float dist2 = length(c - col2);
    float dist3 = length(c - col3);
    float dist4 = length(c - col4);

    float d = min(dist1,dist2);
    d = min(d,dist3);
    d = min(d,dist4);

    if(d == dist1) {
        c = col1;
    }
    else if(d == dist2) {
        c = col2;
    }
    else if(d == dist3) {
        c = col3;
    }
    else {
        c = col4;
    }

    //gl_FragColor = vec4(c,alpha).rgba;
    fragColor = vec4(c,alpha).rgba;
}

here I also included the way to convert this shadertoy shader to a game maker studio shader, because in game maker studio there isn't the uniform time and resolution to make a time editing and the vec2 uv

I also included the commented right palettes for all the cga color palettes and a bonus palette 0 that it is a gameboy palette

I also included the conversion from rgb 255 value to the float for who could not be able to convert

the only problem here is that it is not able to invert colors or change position of the colors in the palette, it takes the original source and output the 4 colors, this is the only gap.