Update document set property using Javascript

Try below code using JSOM:

var folder = context.Web.GetFolderByServerRelativeUrl("/sites/test/Document/DocSetName");  
folder.ListItemAllFields("Title") = "NewDocSetName";
folder.ListItemAllFields("FileLeafRef") = "NewDocSetName";
folder.ListItemAllFields("DocumentSetDescription") = "updated description";
folder.ListItemAllFields.Update();
context.load(folder);
context.ExecuteQuery();

Using REST:

function executeJson(url,method,additionalHeaders,payload)
{
    var headers = {};
    headers["Accept"] = "application/json;odata=verbose";
    if(method == "POST") {
        headers["X-RequestDigest"] = $("#__REQUESTDIGEST").val();
    }
    if (typeof additionalHeaders != 'undefined') {
        for(var key in additionalHeaders){
            headers[key] = additionalHeaders[key];
        }
    }

    var ajaxOptions =
    {
       url: url,
       type: method,
       contentType: "application/json;odata=verbose",
       headers: headers
    };
    if(method == "POST") {
      ajaxOptions.data = JSON.stringify(payload);
    }

    return $.ajax(ajaxOptions);
}

function renameDocSet(webUrl,folderUrl,name)
{
     var folderItemUrl =  webUrl + "/_api/web/GetFolderByServerRelativeUrl('" + folderUrl + "')/ListItemAllFields";
     return executeJson(folderItemUrl,"GET").then(
         function(data){
             var itemPayload = {};
             itemPayload['__metadata'] = {'type': data.d['__metadata']['type']};
             itemPayload['Title'] = name;
             itemPayload['FileLeafRef'] = name;
             itemPayload['DocumentSetDescription']="updated desc"
             var itemUrl = data.d['__metadata']['uri'];
             var additionalHeaders = {};
             additionalHeaders["X-HTTP-Method"] = "MERGE";
             additionalHeaders["If-Match"] =  "*";
             return executeJson(itemUrl,"POST",additionalHeaders,itemPayload);
         });
}

REST api reference - Update document set properties using REST