باید با rtf کار کنید.
این متد رو تست کنید
private void InsertEnter(RichTextBox richtxt)
{
int selectionStartIndex = richtxt.SelectionStart;
int selectionLength = richtxt.SelectionLength;
richtxt.SelectionStart = 0;
richtxt.SelectionLength = selectionStartIndex;
int indexForInsert = richtxt.SelectedRtf.LastIndexOf('}');
string rtfPart1 = richtxt.SelectedRtf;
richtxt.SelectionStart = selectionStartIndex + selectionLength;
richtxt.SelectionLength = richtxt.TextLength - (selectionStartIndex + selectionLength);
string rtfPart2 = richtxt.SelectedRtf;
richtxt.Text = string.Empty;
richtxt.SelectedRtf = rtfPart1.Insert(indexForInsert , "\\par");
richtxt.SelectionStart = richtxt.Text.Length;
richtxt.SelectedRtf = rtfPart2;
richtxt.SelectionStart = selectionStartIndex + 1;
richtxt.Focus();
}