Photoshop JS script to create and apply a layer mask

Question

I wrote a Photoshop script in JS that takes each layer folder and spaces it out like a sprite sheet. I've figured out that part, but I'm trying to remove any form of human error while running the script. Right now, you need to prep your file with the correct naming of each layer folder, and you also have to apply a layer mask to the selection.

I want to remove the need for the user to apply a layer mask. I can select the layer, and then select the portion I want to mask, but I have no clue on how to apply or create the mask.

Where I want it to be applied:

function maskIt(){
    if(currentFrameCount < (frameNumber-1)){
        currentFrameCount = currentFrameCount+1;
        currentFrame = ("frame"+currentFrameCount);
        activeDocument.layers[currentFrame].visable;
        activeDocument.selection.selectAll();
        //createMask();
        maskComplete = false;
    } else  if (currentFrameCount == (frameNumber-1)){
        currentFrameCount = currentFrameCount+1;
        currentFrame = ("frame"+currentFrameCount);
        activeDocument.layers[currentFrame].visable;
        activeDocument.selection.selectAll();
        //createMask();
        currentFrameCount = 0;
        maskComplete = true;
    }
}

Answer 1

Here is a cleaner version with try added to check for the existence of a mask. If you want to apply any mask, just move the line deleteLayerMask(true);below catch (e) {}

addMasks();

function addMasks(){

    try{
        loadLayerSelection();
        addLayerMask();
        deleteLayerMask(true);
        } catch (e) {}
};

// =======================================================
function loadLayerSelection() {
    var c2t = function (s) {
        return app.charIDToTypeID(s);
    };

    var s2t = function (s) {
        return app.stringIDToTypeID(s);
    };

    var descriptor = new ActionDescriptor();
    var reference = new ActionReference();
    var reference2 = new ActionReference();

    reference.putProperty( s2t( "channel" ), s2t( "selection" ));
    descriptor.putReference( c2t( "null" ), reference );
    reference2.putEnumerated( s2t( "channel" ), s2t( "channel" ), s2t( "transparencyEnum" ));
    descriptor.putReference( s2t( "to" ), reference2 );
    executeAction( s2t( "set" ), descriptor, DialogModes.NO );
}

// =======================================================
function addLayerMask() {
    var c2t = function (s) {
        return app.charIDToTypeID(s);
    };

    var s2t = function (s) {
        return app.stringIDToTypeID(s);
    };

    var descriptor = new ActionDescriptor();
    var reference = new ActionReference();

    descriptor.putClass( s2t( "new" ), s2t( "channel" ));
    reference.putEnumerated( s2t( "channel" ), s2t( "channel" ), s2t( "mask" ));
    descriptor.putReference( s2t( "at" ), reference );
    descriptor.putEnumerated( s2t( "using" ), c2t( "UsrM" ), s2t( "revealSelection" ));
    executeAction( s2t( "make" ), descriptor, DialogModes.NO );
}

// =======================================================
function deleteLayerMask(apply) {
    var c2t = function (s) {
        return app.charIDToTypeID(s);
    };

    var s2t = function (s) {
        return app.stringIDToTypeID(s);
    };

    var descriptor = new ActionDescriptor();
    var reference = new ActionReference();

    reference.putEnumerated( s2t( "channel" ), s2t( "channel" ), s2t( "mask" ));
    descriptor.putReference( c2t( "null" ), reference );
    descriptor.putBoolean( s2t( "apply" ), apply );
    executeAction( s2t( "delete" ), descriptor, DialogModes.NO );
}

Answer 2

These two functions tidied up from the scriptlistener should help you:

// FUNCTION MAKE MASK ()
function makeMask()
{
  // =======================================================
  var id4556 = charIDToTypeID( "setd" );
  var desc983 = new ActionDescriptor();
  var id4557 = charIDToTypeID( "null" );
  var ref657 = new ActionReference();
  var id4558 = charIDToTypeID( "Chnl" );
  var id4559 = charIDToTypeID( "fsel" );
  ref657.putProperty( id4558, id4559 );
  desc983.putReference( id4557, ref657 );
  var id4560 = charIDToTypeID( "T   " );
  var ref658 = new ActionReference();
  var id4561 = charIDToTypeID( "Chnl" );
  var id4562 = charIDToTypeID( "Chnl" );
  var id4563 = charIDToTypeID( "Trsp" );
  ref658.putEnumerated( id4561, id4562, id4563 );
  desc983.putReference( id4560, ref658 );
  executeAction( id4556, desc983, DialogModes.NO );

  // =======================================================
  var id4564 = charIDToTypeID( "Mk  " );
  var desc984 = new ActionDescriptor();
  var id4565 = charIDToTypeID( "Nw  " );
  var id4566 = charIDToTypeID( "Chnl" );
  desc984.putClass( id4565, id4566 );
  var id4567 = charIDToTypeID( "At  " );
  var ref659 = new ActionReference();
  var id4568 = charIDToTypeID( "Chnl" );
  var id4569 = charIDToTypeID( "Chnl" );
  var id4570 = charIDToTypeID( "Msk " );
  ref659.putEnumerated( id4568, id4569, id4570 );
  desc984.putReference( id4567, ref659 );
  var id4571 = charIDToTypeID( "Usng" );
  var id4572 = charIDToTypeID( "UsrM" );
  var id4573 = charIDToTypeID( "RvlS" );
  desc984.putEnumerated( id4571, id4572, id4573 );
  executeAction( id4564, desc984, DialogModes.NO );
}


// FUNCTION APPLY LAYER MASK()
function applyLayerMask()
{
  // =======================================================
  var id1949 = charIDToTypeID( "Dlt " );
  var desc398 = new ActionDescriptor();
  var id1950 = charIDToTypeID( "null" );
  var ref291 = new ActionReference();
  var id1951 = charIDToTypeID( "Chnl" );
  var id1952 = charIDToTypeID( "Chnl" );
  var id1953 = charIDToTypeID( "Msk " );
  ref291.putEnumerated( id1951, id1952, id1953 );
  desc398.putReference( id1950, ref291 );
  var id1954 = charIDToTypeID( "Aply" );
  desc398.putBoolean( id1954, true );
  executeAction( id1949, desc398, DialogModes.NO );
}

If you are trying to rule out user error human you may need a third function which will detect whether a layer has as mask on or not (Select that layer, try to copy and paste the layer mask; if it doesn't paste - no layer mask)

Answer 3

A much cleaner way to apply masks (clipping and/or layer) to the currently selected object:

// Clipping Mask
const applyClippingMask = function () {
    // Grab the needed IDs
    const groupEventID = stringIDToTypeID('groupEvent')
    const nullID = stringIDToTypeID('null')
    const layerID = stringIDToTypeID('layer')
    const ordinalID = stringIDToTypeID('ordinal')
    const targetEnumID = stringIDToTypeID('targetEnum')

    // Prep the action
    const actionDesc = new ActionDescriptor()
    const actionRef = new ActionReference()

    // Prep the mask
    actionRef.putEnumerated(layerID, ordinalID, targetEnumID)
    actionDesc.putReference(nullID, actionRef)
    
    // Apply the mask
    executeAction(groupEventID, actionDesc, DialogModes.NO)
}

// Layer mask from selection
const applyLayerMask = function () {
    // Required internal IDs
    const makeID = stringIDToTypeID('make')
    const newID = stringIDToTypeID('new')
    const channelID = stringIDToTypeID('channel')
    const atID = stringIDToTypeID('at')
    const usingID = stringIDToTypeID('using')
    const userMaskEnabledID = stringIDToTypeID('userMaskEnabled')
    const revealSelectionID = stringIDToTypeID('revealSelection')

    // Prep the action
    const actionDesc = new ActionDescriptor()
    const actionRef = new ActionReference()

    // Setup the mask
    actionDesc.putClass(newID, channelID)
    actionRef.putEnumerated(channelID, channelID, maskID)
    actionDesc.putReference(atID, actionRef)
    actionDesc.putEnumerated(usingID, userMaskEnabledID, revealSelectionID)

    // Apply the mask
    executeAction(makeID, actionDesc, DialogModes.NO)
}

Edited to include both clipping and layer mask functions based on comment from Sergey

Works in Photoshop CC 2021