glx: force glx usage within the same X11 window instance

This commit fixes a crash on Debian Stretch and Ubuntu 17.10 when you fallback
from VDPAU (GLX forced) to SW decoding (EGL by default).

Fixes #19982

modules/video_output/glx.c

@@ -237,6 +237,16 @@ static int Open (vlc_object_t *obj)
     ReleaseCurrent (gl);
 #endif
 
+    /* XXX: Prevent other gl backends (like EGL) to be opened within the same
+     * X11 window instance. Indeed, using EGL after GLX on the same X11 window
+     * instance leads to an SEGFAULT in the libEGL_nvidia.so library. */
+    const char *vendor = glXGetClientString(dpy, GLX_VENDOR);
+    if (vendor && strncmp(vendor, "NVIDIA", sizeof("NVIDIA") - 1) == 0)
+    {
+        var_Create(gl->surface, "gl", VLC_VAR_STRING);
+        var_SetString(gl->surface, "gl", "glx");
+    }
+
     return VLC_SUCCESS;
 
 error: