Нашел вариант задания правил, при котором все работает.
<contexts>
<context name="Normal" attribute="Normal Text" lineEndContext="#stay">
<RegExpr attribute="Operator" context="Quote operator brace" String="\bq'\{" insensitive="true"/>
<RegExpr attribute="Operator" context="Quote operator bracket" String="\bq'\[" insensitive="true"/>
<RegExpr attribute="Operator" context="Quote operator parenthesis" String="\bq'\(" insensitive="true"/>
<RegExpr attribute="Operator" context="Quote operator" String="\bq'(\S)" insensitive="true"/>
<DetectChar attribute="String" context="String literal" char="'" insensitive="true"/>
</context>
<context name="Quote operator" attribute="String" lineEndContext="#stay" dynamic="true">
<RegExpr attribute="Operator" context="#pop" String="%1'" dynamic="true"/>
</context>
<context name="Quote operator brace" attribute="String" lineEndContext="#stay">
<Detect2Chars attribute="Operator" context="#pop" char="}" char1="'"/>
</context>
<context name="Quote operator bracket" attribute="String" lineEndContext="#stay">
<Detect2Chars attribute="Operator" context="#pop" char="]" char1="'"/>
</context>
<context name="Quote operator parenthesis" attribute="String" lineEndContext="#stay">
<Detect2Chars attribute="Operator" context="#pop" char=")" char1="'"/>
</context>
</contexts>