Отправьте сообщение в процесс Windows (не его главное окно)

Для других людей, желающих этого добиться, я публикую ниже свою реализацию, используя решение Мэтта Дэвиса.

В Program.cs

static class Program
{
    #region Dll Imports
    public const int HWND_BROADCAST = 0xFFFF;

    [DllImport("user32.dll")]
    public static extern bool SetForegroundWindow(IntPtr hWnd);

    [DllImport("user32")]
    public static extern bool PostMessage(IntPtr hwnd, int msg, IntPtr wparam, IntPtr lparam);

    [DllImport("user32")]
    public static extern int RegisterWindowMessage(string message);
    #endregion Dll Imports

    public static readonly int WM_ACTIVATEAPP = RegisterWindowMessage("WM_ACTIVATEAPP");

    [STAThread]
    static void Main()
    {
        bool createdNew = true;
        //by creating a mutex, the next application instance will detect it
        //and the code will flow through the "else" branch 
        using (Mutex mutex = new Mutex(true, "MyMutexName", out createdNew))//make sure it's an unique identifier (a GUID would be better)
        {
            if (createdNew)
            {
                Application.EnableVisualStyles();
                Application.SetCompatibleTextRenderingDefault(false);
                Application.Run(new MainForm());
            }
            else
            {
                //we tried to create a mutex, but there's already one (createdNew = false - another app created it before)
                //so there's another instance of this application running
                Process currentProcess = Process.GetCurrentProcess();

                //get the process that has the same name as the current one but a different ID
                foreach (Process process in Process.GetProcessesByName(currentProcess.ProcessName))
                {
                    if (process.Id != currentProcess.Id)
                    {
                        IntPtr handle = process.MainWindowHandle;

                        //if the handle is non-zero then the main window is visible (but maybe somewhere in the background, that's the reason the user started a new instance)
                        //so just bring the window to front
                        if (handle != IntPtr.Zero)
                            SetForegroundWindow(handle);
                        else
                            //tough luck, can't activate the window, it's not visible and we can't get its handle
                            //so instead notify the process that it has to show it's window
                            PostMessage((IntPtr)HWND_BROADCAST, WM_ACTIVATEAPP, IntPtr.Zero, IntPtr.Zero);//this message will be sent to MainForm

                        break;
                    }
                }
            }
        }
    }
}

В MainForm.cs

protected override void WndProc(ref Message m)
{
    base.WndProc(ref m);
            //someone (another process) said that we should show the window (WM_ACTIVATEAPP)
    if (m.Msg == Program.WM_ACTIVATEAPP)
        this.Show();
}

Для этого можно использовать именованные каналы. Это может быть более приемлемый метод с .net. Вы можете определить службу в главном приложении, которая принимает сообщение от вызывающего приложения. Вот образец услуги в vb. Он вызывает основное приложение и передает ему строку, в данном случае имя файла. Он также возвращает строку, но здесь можно использовать любые параметры.

Public Class PicLoadService : Implements IMainAppPicLoad

Public Function LoadPic(ByVal fName As String) As String Implements IMainAppPicLoad.LoadPic
' do some stuff here.
LoadPic = "return string"
End Function

End Class

Настройка в вызывающем приложении немного сложнее. Вызывающее и основное приложение могут быть одним и тем же приложением.

Imports System.Diagnostics
Imports System.ServiceModel
Imports System.IO
Imports vb = Microsoft.VisualBasic

Module MainAppLoader

Sub Main()

Dim epAddress As EndpointAddress
Dim Client As picClient
Dim s As String
Dim loadFile As String
Dim procs() As Process
Dim processName As String = "MainApp"

loadFile = "" ' filename to load

procs = Process.GetProcessesByName(processName)

If UBound(procs) >= 0 Then
  epAddress = New EndpointAddress("net.pipe://localhost/MainAppPicLoad")
  Client = New picClient(New NetNamedPipeBinding, epAddress)
  s = Client.LoadPic(loadFile)
End If

End Sub

<System.Diagnostics.DebuggerStepThroughAttribute(), _
 System.CodeDom.Compiler.GeneratedCodeAttribute("System.ServiceModel", "3.0.0.0")> _
Partial Public Class picClient
    Inherits System.ServiceModel.ClientBase(Of IMainAppPicLoad)
    Implements IMainAppPicLoad

    Public Sub New(ByVal binding As System.ServiceModel.Channels.Binding, ByVal remoteAddress As System.ServiceModel.EndpointAddress)
        MyBase.New(binding, remoteAddress)
    End Sub

    Public Function LoadPic(ByVal fName As String) As String Implements IMainAppPicLoad.LoadPic
        Return MyBase.Channel.LoadPic(fName)
    End Function

End Class

' from here down was auto generated by svcutil.
' svcutil.exe /language:vb /out:generatedProxy.vb /config:app.config http://localhost:8000/MainAppPicLoad
' Some has been simplified after auto code generation.
<System.CodeDom.Compiler.GeneratedCodeAttribute("System.ServiceModel", "3.0.0.0"), _
 System.ServiceModel.ServiceContractAttribute(ConfigurationName:="IMainAppPicLoad")> _
Public Interface IMainAppPicLoad
  <System.ServiceModel.OperationContractAttribute(Action:="http://tempuri.org/IMainAppPicLoad/LoadPic", ReplyAction:="http://tempuri.org/IMainAppPicLoad/LoadPicResponse")> _
  Function LoadPic(ByVal fName As String) As String
End Interface

<System.CodeDom.Compiler.GeneratedCodeAttribute("System.ServiceModel", "3.0.0.0")> _
Public Interface IMainAppPicLoadChannel
  Inherits IMainAppPicLoad, System.ServiceModel.IClientChannel
End Interface

<System.Diagnostics.DebuggerStepThroughAttribute(), _
System.CodeDom.Compiler.GeneratedCodeAttribute("System.ServiceModel", "3.0.0.0")> _
Partial Public Class IMainAppPicLoadClient
  Inherits System.ServiceModel.ClientBase(Of IMainAppPicLoad)
  Implements IMainAppPicLoad

  Public Sub New(ByVal binding As System.ServiceModel.Channels.Binding, ByVal remoteAddress As System.ServiceModel.EndpointAddress)
    MyBase.New(binding, remoteAddress)
  End Sub

  Public Function LoadPic(ByVal fName As String) As String Implements IMainAppPicLoad.LoadPic
    Return MyBase.Channel.LoadPic(fName)
  End Function
End Class

End Module

<ServiceContract()> Public Interface IMainAppPicLoad
<OperationContract()> Function LoadPic(ByVal fName As String) As String
End Interface