TYPO3 Core

Yes, TypoScript is OK. Even if it was not, the RETURN_URL-marker is still not replaced correctly.

I can confirm this bug. I already spend several hours of debugging and i thing i found the problem:
Replacement is not working, because the menuTypoLink function in class.tslib_menu.php do not get a result from the typolink call on pages with restrictions.
Event not the lastTypoLinkUrl brings any result back.
My solution is to set the typolink configuration "linkAccessRestrictedPages", before generating the link.
To prevent double adding additional parameter, if the config.typolinkLinkAccessRestrictedPages is also enabled, some more thinks must be observed.

Here my suggestion how to fix (a patch will follow soon):

Line 1152:

            // Creating link:
        if ($this->mconf['collapse'] && $this->isActive($this->menuArr[$key]['uid'], $this->getMPvar($key)))    {
            $thePage = $this->sys_page->getPage($this->menuArr[$key]['pid']);
            $addParams = $this->mconf['addParams'].$MP_params.$this->menuArr[$key]['_ADD_GETVARS'];
            // in2code: bugfix for link generation of restricted pages
            $showAccessRestrictedPages = $this->mconf['showAccessRestrictedPages']
                    && $this->mconf['showAccessRestrictedPages']!=='NONE'
                    && !$GLOBALS['TSFE']->checkPageGroupAccess($thePage);
            if ($GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages']
                    && $GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages'] !== 'NONE'
                    && !$GLOBALS['TSFE']->checkPageGroupAccess($thePage)) {
                // temporary deactivate typolinkLinkAccessRestrictedPages to disable double ###RETURN_URL### and ###PAGE_ID### replacement
                $tempTypolinkLinkAccessRestrictedPages = $GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages'];
                unset($GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages']);
            }
            $LD = $this->menuTypoLink($thePage,$mainTarget,'','',$overrideArray, $addParams, $typeOverride, $showAccessRestrictedPages);
            $GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages'] = $tempTypolinkLinkAccessRestrictedPages;
        } else {
            $addParams = $this->mconf['addParams'].$MP_params.$this->I['val']['additionalParams'].$this->menuArr[$key]['_ADD_GETVARS'];
            // in2code: bugfix for link generation of restricted pages
            $showAccessRestrictedPages = $this->mconf['showAccessRestrictedPages']
                    && $this->mconf['showAccessRestrictedPages']!=='NONE'
                    && !$GLOBALS['TSFE']->checkPageGroupAccess($this->menuArr[$key]);
            if ($GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages']
                    && $GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages'] !== 'NONE'
                    && !$GLOBALS['TSFE']->checkPageGroupAccess($this->menuArr[$key])) {
                // temporary deactivate typolinkLinkAccessRestrictedPages to disable double ###RETURN_URL### and ###PAGE_ID### replacement
                $tempTypolinkLinkAccessRestrictedPages = $GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages'];
                unset($GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages']);
            }
            $LD = $this->menuTypoLink($this->menuArr[$key],$mainTarget,'','',$overrideArray, $addParams, $typeOverride, $showAccessRestrictedPages);
            $GLOBALS['TSFE']->config['config']['typolinkLinkAccessRestrictedPages'] = $tempTypolinkLinkAccessRestrictedPages;
        }

Line 1620:

    /**
     * Calls typolink to create menu item links.
     *
     * @param    array        $page    Page record (uid points where to link to)
     * @param    string        $oTarget    Target frame/window
     * @param    boolean        $no_cache    TRUE if caching should be disabled
     * @param    string        $script    Alternative script name
     * @param    array        $overrideArray    Array to override values in $page
     * @param    string        $addParams    Parameters to add to URL
     * @param    array        $typeOverride    "type" value
     * @param    boolean        $enableRestrictedPages    Enable link generation for restricted pages
     * @return    array        See linkData
     */
    function menuTypoLink($page, $oTarget, $no_cache, $script, $overrideArray = '', $addParams = '', $typeOverride = '', $enableRestrictedPages = false) {
        $conf = array(
            'parameter' => is_array($overrideArray) && $overrideArray['uid'] ? $overrideArray['uid'] : $page['uid'],
        );
        if ($typeOverride && t3lib_utility_Math::canBeInterpretedAsInteger($typeOverride)) {
            $conf['parameter'] .= ',' . $typeOverride;
        }
        if ($addParams) {
            $conf['additionalParams'] = $addParams;
        }
        if ($no_cache) {
            $conf['no_cache'] = TRUE;
        }
        if ($oTarget) {
            $conf['target'] = $oTarget;
        }
        if ($page['sectionIndex_uid']) {
            $conf['section'] = $page['sectionIndex_uid'];
        }
        if ($enableRestrictedPages) {
            $conf['linkAccessRestrictedPages'] = TRUE;
        }

        $this->parent_cObj->typoLink('|', $conf);
        $LD = $this->parent_cObj->lastTypoLinkLD;

        $LD['totalURL'] = $this->parent_cObj->lastTypoLinkUrl;
        return $LD;
    }