I think you didn't make a fair translation from ruby to javascript there, here goes my attempt:

    function map_unless_nested(arr){                                                                      
        return arr.filter(function(it){ if (it instanceof Array) return it }).length?
            function(){ return [] } : function(fn){ return arr.map(fn) };
    }

    console.log(map_unless_nested([1,2,3])(function(n){ return n + 1 }));
    console.log(map_unless_nested([1,2,[1,2],3])(function(n){ return n + 1 }));

edit: also a translation for the last example

    function map_first_innermost(arr){
        return function(fn){
            return arr.map(function(v){
                if (v instanceof Array) 
                    return map_first_innermost(v)(fn);       
                return fn(v);
            });
        };
    }

    console.log(map_first_innermost([1,2,[1,2],3])(function(n){ return n + 1 }));